From 862c857f0502006d8cc12094ecdf406dc52b7d54 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 10:06:10 +0000 Subject: [PATCH 1/3] Phase 2 coordinator documents: pipeline map, stage contracts, call graphs, lifecycle, spec draft, quirks, design space, corpus Adds 00-pipeline-map, STAGE-CONTRACTS, CALL-GRAPHS (generated), LIFECYCLE-script-execution, LANGUAGE-SPEC-DRAFT, SEMANTIC-QUIRKS (32 quirks), INTERPRETER-DESIGN-SPACE (scores computed from data/interpreter-scores.json), GOLDEN-SAMPLES (empty), ORACLE-PROTOCOL (waived), REFERENCE-TEST-CORPUS (generated), design-canvas skeleton and docs-out/README.md. Corrects the execution-entry statement in 02.5/TIERING-POLICY/SCRIPT-BLOCKS (CON-013). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01HnR7RpZuh9mvNMRTtzxTvk --- docs-out/.work/cf/COORD.json | 16 + docs-out/.work/cf/T27.json | 5 +- docs-out/.work/claims/COORD.jsonl | 8 + docs-out/.work/numbers-proposals/COORD.json | 65 +++ docs-out/02-language/00-pipeline-map.md | 48 ++ .../02-language/02.5-compiler-and-engine.md | 4 +- docs-out/02-language/CALL-GRAPHS.md | 356 +++++++++++++ docs-out/02-language/GOLDEN-SAMPLES.md | 97 ++++ .../02-language/INTERPRETER-DESIGN-SPACE.md | 237 +++++++++ docs-out/02-language/LANGUAGE-SPEC-DRAFT.md | 127 +++++ .../02-language/LIFECYCLE-script-execution.md | 158 ++++++ docs-out/02-language/ORACLE-PROTOCOL.md | 100 ++++ docs-out/02-language/REFERENCE-TEST-CORPUS.md | 242 +++++++++ docs-out/02-language/SCRIPT-BLOCKS.md | 2 +- docs-out/02-language/SEMANTIC-QUIRKS.md | 132 +++++ docs-out/02-language/STAGE-CONTRACTS.md | 181 +++++++ docs-out/02-language/TIERING-POLICY.md | 6 +- docs-out/09-synthesis/deviation-register.md | 3 +- .../new-language-design-canvas.md | 40 ++ docs-out/CONTRADICTIONS.md | 1 + docs-out/DECISIONS.md | 1 + docs-out/PROGRESS.md | 8 +- docs-out/README.md | 20 + docs-out/data/canonical-facts.json | 5 +- docs-out/data/claims/COORD.jsonl | 8 + docs-out/data/interpreter-scores.json | 473 +++++++++++++++++- docs-out/data/lessons.json | 235 ++++++++- docs-out/data/numbers-extra.json | 63 +++ docs-out/data/numbers.json | 81 +++ docs-out/data/subject-keys.json | 8 + .../__pycache__/gen_coord.cpython-313.pyc | Bin 0 -> 30895 bytes docs-out/tools/gen_coord.py | 346 +++++++++++++ 32 files changed, 3059 insertions(+), 17 deletions(-) create mode 100644 docs-out/.work/cf/COORD.json create mode 100644 docs-out/.work/claims/COORD.jsonl create mode 100644 docs-out/.work/numbers-proposals/COORD.json create mode 100644 docs-out/02-language/00-pipeline-map.md create mode 100644 docs-out/02-language/CALL-GRAPHS.md create mode 100644 docs-out/02-language/GOLDEN-SAMPLES.md create mode 100644 docs-out/02-language/INTERPRETER-DESIGN-SPACE.md create mode 100644 docs-out/02-language/LANGUAGE-SPEC-DRAFT.md create mode 100644 docs-out/02-language/LIFECYCLE-script-execution.md create mode 100644 docs-out/02-language/ORACLE-PROTOCOL.md create mode 100644 docs-out/02-language/REFERENCE-TEST-CORPUS.md create mode 100644 docs-out/02-language/SEMANTIC-QUIRKS.md create mode 100644 docs-out/02-language/STAGE-CONTRACTS.md create mode 100644 docs-out/09-synthesis/new-language-design-canvas.md create mode 100644 docs-out/README.md create mode 100644 docs-out/data/claims/COORD.jsonl create mode 100644 docs-out/tools/__pycache__/gen_coord.cpython-313.pyc create mode 100644 docs-out/tools/gen_coord.py diff --git a/docs-out/.work/cf/COORD.json b/docs-out/.work/cf/COORD.json new file mode 100644 index 0000000..4ca39e0 --- /dev/null +++ b/docs-out/.work/cf/COORD.json @@ -0,0 +1,16 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-DESIGN-110", + "baseline_behavior": "PowerShell compiles to expression trees and uses Reflection.Emit for classes.", + "proposed_behavior": "MicroteamShell starts with a tree-walking evaluator and does not adopt Reflection.Emit or Expression.Compile as the core engine.", + "benefit": "AOT-friendly, debuggable, small.", + "compatibility_cost": "Lower peak throughput; classes (if added) are interpreter objects.", + "affected_cfs": [ + "CF-CMP-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T27.json b/docs-out/.work/cf/T27.json index aafe823..d741b17 100644 --- a/docs-out/.work/cf/T27.json +++ b/docs-out/.work/cf/T27.json @@ -31,13 +31,14 @@ { "id": "CF-CMP-003", "subject": "optimized-versus-unoptimized preconditions", - "statement": "**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used for script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables.", + "statement": "**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used by the command processors that run script blocks (`DlrScriptCommandProcessor` constructor) and script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables.", "label": "FACT-SRC", "evidence": [ "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83", "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177", "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552", - "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847" + "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847", + "src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.Init@L274-L296" ], "owner": "02-language/TIERING-POLICY.md", "status": "open" diff --git a/docs-out/.work/claims/COORD.jsonl b/docs-out/.work/claims/COORD.jsonl new file mode 100644 index 0000000..8074b42 --- /dev/null +++ b/docs-out/.work/claims/COORD.jsonl @@ -0,0 +1,8 @@ +{"id": "CLM-LIFE-0001", "subject_key": "lifecycle.script_parse_path", "predicate": "script command text is parsed by", "value": "AutomationEngine.ParseScriptBlock via EngineParser.Parse (no script block cache)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0002", "subject_key": "lifecycle.incomplete_input", "predicate": "a parse error that is incomplete input raises", "value": "IncompleteParseException", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0003", "subject_key": "lifecycle.nolanguage_scripts", "predicate": "script text with NoLanguage and runspace origin", "value": "rejected (ScriptsNotAllowed)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0004", "subject_key": "lifecycle.cmdletbinding_processor", "predicate": "script block using cmdlet binding runs in", "value": "CommandProcessor over a FunctionInfo", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0005", "subject_key": "lifecycle.script_processor", "predicate": "other script blocks run in", "value": "DlrScriptCommandProcessor", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor@L232-L625"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0006", "subject_key": "lifecycle.flowcontrol_across_commands", "predicate": "FlowControlException in a clause", "value": "rethrown out of the command processor", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L625"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0007", "subject_key": "lifecycle.exit_script_file", "predicate": "ExitException from a script file", "value": "sets LASTEXITCODE and marks pipeline failed when non-zero", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L625"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0008", "subject_key": "lifecycle.outdefault_appended", "predicate": "executor appends Out-Default for rendering when", "value": "AddOutputter option set", "label": "FACT-SRC", "evidence": ["src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor.ExecuteCommandHelper@L335-L357"], "doc": "LANG-LIFECYCLE-001", "cf": null} diff --git a/docs-out/.work/numbers-proposals/COORD.json b/docs-out/.work/numbers-proposals/COORD.json new file mode 100644 index 0000000..20b1aa8 --- /dev/null +++ b/docs-out/.work/numbers-proposals/COORD.json @@ -0,0 +1,65 @@ +[ + { + "key": "PARSER_RULE_NODES", + "cmd": "python3 docs-out/tools/gen_coord.py --count callgraph_nodes", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_TOTAL", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_total", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_REPLICATE", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_replicate", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_ADAPT", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_adapt", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_AVOID", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_avoid", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_REDESIGN", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_redesign", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "LANGUAGE_TEST_FILES", + "cmd": "python3 docs-out/tools/gen_coord.py --count test_files", + "dataset": "tests.json", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "LANGUAGE_TEST_ITS", + "cmd": "python3 docs-out/tools/gen_coord.py --count test_its", + "dataset": "tests.json", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "INTERPRETER_OPTIONS", + "cmd": "python3 docs-out/tools/gen_coord.py --count options", + "dataset": "interpreter-scores.json", + "label": "APPROX", + "notes": "coordinator documents" + } +] \ No newline at end of file diff --git a/docs-out/02-language/00-pipeline-map.md b/docs-out/02-language/00-pipeline-map.md new file mode 100644 index 0000000..05ef924 --- /dev/null +++ b/docs-out/02-language/00-pipeline-map.md @@ -0,0 +1,48 @@ +--- +id: LANG-MAP-001 +title: Language pipeline map +kind: guide +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/ast.cs, src/System.Management.Automation/engine/parser/Compiler.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [] +--- + +# Language pipeline map + +> STATIC-ONLY — behavior not run-verified. One page: stage → files → entry points → data handed to the next stage. Details: [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md) (exact contracts), [CALL-GRAPHS.md](CALL-GRAPHS.md), [LIFECYCLE-script-execution.md](LIFECYCLE-script-execution.md). + +```mermaid +flowchart LR + T[text] --> TK[tokenize] + TK --> P[parse] + P --> A[AST] + A --> C[check] + C --> CP[compile] + CP --> IR[expression trees] + IR --> TI[interpret or JIT] + TI --> V[values and output pipe] + TK -. mode requests .-> P +``` + +| # | Stage | Files (under `src/System.Management.Automation/engine/`) | Entry point | Hands to next stage | Document | +|---|---|---|---|---|---| +| 1 | Tokenize | `parser/tokenizer.cs`, `parser/token.cs`, `parser/CharTraits.cs` | `Tokenizer.NextToken` (pull, mode-driven) | `Token` objects, optional token list | [02.1-tokenizer.md](02.1-tokenizer.md), [TOKEN-CATALOG.md](TOKEN-CATALOG.md) | +| 2 | Parse | `parser/Parser.cs` | `Parser.ParseInput` / `ParseFile` | `ScriptBlockAst`, `Token[]`, `ParseError[]` | [02.2-parser.md](02.2-parser.md), [GRAMMAR.ebnf.md](GRAMMAR.ebnf.md), [ERROR-CATALOG.md](ERROR-CATALOG.md) | +| 3 | AST | `parser/ast.cs`, `parser/AstVisitor.cs`, `parser/PreOrderVisitor.cs` | `new …Ast(…)` in the parser | the tree (parents, extents) | [02.3-ast.md](02.3-ast.md), [AST-CATALOG.md](AST-CATALOG.md) | +| 4 | Check | `parser/SymbolResolver.cs`, `parser/SemanticChecks.cs`, `parser/TypeResolver.cs`, `parser/VariableAnalysis.cs` | `ScriptBlockAst.PerformPostParseChecks` | extra `ParseError`s; node annotations | [02.4-semantic-analysis.md](02.4-semantic-analysis.md) | +| 5 | Compile | `parser/Compiler.cs`, `runtime/CompiledScriptBlock.cs` | `CompiledScriptBlockData.Compile` → `Compiler.Compile` | per-block `Action` (tiered delegate) | [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md), [COMPILATION-MAP.md](COMPILATION-MAP.md), [IR-SPEC.md](IR-SPEC.md) | +| 6 | Tier | `interpreter/*.cs` | `LightCompiler.CompileTop`, `LightLambda` | interpreted or JIT code | [TIERING-POLICY.md](TIERING-POLICY.md), [INSTRUCTION-SET.md](INSTRUCTION-SET.md), [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md) | +| 7 | Execute | `runtime/Binding/Binders.cs`, `runtime/Operations/*.cs`, `LanguagePrimitives.cs`, session state | `ScriptBlock.InvokeWithPipeImpl` | values, errors, output-pipe writes | [02.6-types-and-operators.md](02.6-types-and-operators.md), [BINDER-CATALOG.md](BINDER-CATALOG.md), [RUNTIME-OPERATIONS.md](RUNTIME-OPERATIONS.md), [SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md) | + +Cross-cutting: scopes and variables ([SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md)), script blocks ([SCRIPT-BLOCKS.md](SCRIPT-BLOCKS.md)), language features ([02.7-language-features.md](02.7-language-features.md)), tooling ([02.8-language-tooling.md](02.8-language-tooling.md)), extension ([HOWTO-extend-language.md](HOWTO-extend-language.md)). + +Two facts that shape the map: the **tokenizer is not a separate pass** (the parser pulls tokens and sets the tokenizer mode per rule, [02.1-tokenizer.md](02.1-tokenizer.md) §6.2), and PowerShell does **not** interpret the AST directly: it compiles the AST to expression trees and then interprets or JIT-compiles those ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §1). + +Verification: stage names and entry points taken from the documents linked in each row; file lists from `data/files.csv` areas. diff --git a/docs-out/02-language/02.5-compiler-and-engine.md b/docs-out/02-language/02.5-compiler-and-engine.md index bf16817..929e9c2 100644 --- a/docs-out/02-language/02.5-compiler-and-engine.md +++ b/docs-out/02-language/02.5-compiler-and-engine.md @@ -44,7 +44,7 @@ Responsible for: compiling named blocks, parameter metadata and attributes, per- - `CompiledScriptBlockData.Compile(bool optimized)` is the entry used at run time; it calls `new Compiler().Compile(data, optimize)` once per variant under a lock (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.ReallyCompile@L163-L195), after `PerformSecurityChecks` (AMSI scan for top-level script blocks, suspicious-content flag). - `Compiler.Compile(data, optimize)`: variable analysis → lambdas → `CompileTree` per named block (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964). - Script block creation: `ScriptBlock.Create(text)` parses (`ParseException` on errors), caches by (file name, text) in `s_cachedScripts` (cap 1024, cleared completely when exceeded; skipped for scripts with `using module/assembly` or dynamic keywords) and returns a `Clone()` that shares the compiled data (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.CacheScriptBlock@L609-L637). -- `ScriptBlock.InvokeWithPipeImpl` is the one execution entry (functions, scripts, `&`, `.`, `Invoke`) (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.InvokeWithPipeImpl@L935-L959). +- **Two execution entries share one set of rules.** (1) Command invocation of script text, functions and scripts goes through a command processor: `DlrScriptCommandProcessor` for ordinary script blocks, `CommandProcessor` + `PSScriptCmdlet` for advanced functions; the `DlrScriptCommandProcessor` constructor calls `ScriptBlock.Compile(optimized: debugging off && new scope)` and `RunClause` runs each clause (src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor@L232-L625, see [LIFECYCLE-script-execution.md](LIFECYCLE-script-execution.md) §6). (2) The direct API `ScriptBlock.Invoke*` runs `ScriptBlock.InvokeWithPipeImpl` (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.InvokeWithPipeImpl@L961-L1263). Both choose the variant with the same two conditions, create the locals tuple, switch the language mode and set `$_`/`$input`; the details below are read from entry (2) and checked against entry (1). ## 5. Data model @@ -82,7 +82,7 @@ sequenceDiagram end ``` -`InvokeWithPipeImpl` order (`FACT-SRC`, notes W4): a `Clean` clause is not supported through this entry (`PSNotSupportedException`); a missing clause returns silently; pick the clause; decide `runOptimized` (CF-CMP-003); create the locals tuple (`MakeLocalsTuple`); set `$_`, `$input`, `$this`; set `$PSScriptRoot`/`$PSCommandPath`; apply language-mode switch rules; set up scope (new local scope with the locals tuple, or dot-source scope pushed onto `DottedScopes`); bind arguments (`BindArgumentsForScriptblockInvoke`); build the `FunctionContext`; log start; call the delegate in `try/finally`; restore language mode, pipes, scope, backed-up variables. +`InvokeWithPipeImpl` order (the API entry; the command-processor entry does the same steps in `Prepare`, `DoBegin`/`ProcessRecord`/`Complete` and `RunClause`) (`FACT-SRC`, notes W4): a `Clean` clause is not supported through this entry (`PSNotSupportedException`); a missing clause returns silently; pick the clause; decide `runOptimized` (CF-CMP-003); create the locals tuple (`MakeLocalsTuple`); set `$_`, `$input`, `$this`; set `$PSScriptRoot`/`$PSCommandPath`; apply language-mode switch rules; set up scope (new local scope with the locals tuple, or dot-source scope pushed onto `DottedScopes`); bind arguments (`BindArgumentsForScriptblockInvoke`); build the `FunctionContext`; log start; call the delegate in `try/finally`; restore language mode, pipes, scope, backed-up variables. ### 6.2 Statement lists and error handling diff --git a/docs-out/02-language/CALL-GRAPHS.md b/docs-out/02-language/CALL-GRAPHS.md new file mode 100644 index 0000000..a477770 --- /dev/null +++ b/docs-out/02-language/CALL-GRAPHS.md @@ -0,0 +1,356 @@ +--- +id: LANG-CALLG-001 +title: Call graphs +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/members-tierA.json] +--- + +# Call graphs + +> STATIC-ONLY — behavior not run-verified. Every graph is `GENERATED` by `docs-out/tools/gen_coord.py` from `data/members-tierA.json` and the member line ranges (`APPROX`: an edge A → B means the name `B` followed by `(` or `<` appears in the body of `A`, so overloads and same-named members of other types can create false edges, and calls through delegates or interfaces are missing). Use them as maps, then read the method. Sequence-level narrative: [LIFECYCLE-script-execution.md](LIFECYCLE-script-execution.md); stage data contracts: [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md). + +## 1. Purpose and scope [SPEC] + +Entry-point call graphs for **tokenize**, **parse**, **check**, **compile** and **invoke**. 'Check' is not generated (the semantic passes are visitors, entered through `PerformPostParseChecks`) and is given as a hand-written chain in §5. + +## 2. Source map + +| graph | entry | file | +|---|---|---| +| tokenize | `Tokenizer.NextToken` | `src/System.Management.Automation/engine/parser/tokenizer.cs` | +| parse | `Parser.ParseTask` → `ScriptBlockRule` | `src/System.Management.Automation/engine/parser/Parser.cs` | +| compile | `Compiler.Compile` | `src/System.Management.Automation/engine/parser/Compiler.cs` | +| invoke | `ScriptBlock.InvokeWithPipeImpl` | `src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs` | + +## 3. Tokenize + +`NextToken` and its scanners, depth 3 (`Tokenizer`, name-based). Nodes: 40, edges: 169. + +```mermaid +flowchart LR + Backtick --> ScanUnicodeEscape + CheckOperatorInCommandMode --> GetStringBuilder + CheckOperatorInCommandMode --> InCommandMode + CheckOperatorInCommandMode --> NewToken + CheckOperatorInCommandMode --> PeekChar + CheckOperatorInCommandMode --> ScanGenericToken + NewCommentToken --> CurrentExtent + NewCommentToken --> SaveToken + NewFileRedirectionToken --> CurrentExtent + NewFileRedirectionToken --> InExpressionMode + NewFileRedirectionToken --> NewNumberToken + NewFileRedirectionToken --> SaveToken + NewFileRedirectionToken --> UngetChar + NewInputRedirectionToken --> CurrentExtent + NewInputRedirectionToken --> SaveToken + NewMergingRedirectionToken --> CurrentExtent + NewMergingRedirectionToken --> SaveToken + NewNumberToken --> CurrentExtent + NewNumberToken --> SaveToken + NewToken --> CurrentExtent + NewToken --> SaveToken + NextToken --> AtEof + NextToken --> Backtick + NextToken --> CheckOperatorInCommandMode + NextToken --> GetChar + NextToken --> GetStringBuilder + NextToken --> InCommandMode + NextToken --> InExpressionMode + NextToken --> NewFileRedirectionToken + NextToken --> NewInputRedirectionToken + NextToken --> NewMergingRedirectionToken + NextToken --> NewScriptExtent + NextToken --> NewToken + NextToken --> NormalizeCRLF + NextToken --> PeekChar + NextToken --> ReportError + NextToken --> ReportIncompleteInput + NextToken --> SaveToken + NextToken --> ScanBlockComment + NextToken --> ScanDot + NextToken --> ScanGenericToken + NextToken --> ScanHereStringExpandable + NextToken --> ScanHereStringLiteral + NextToken --> ScanIdentifier + NextToken --> ScanLabel + NextToken --> ScanLineComment + NextToken --> ScanNumber + NextToken --> ScanNumberHelper + NextToken --> ScanParameter + NextToken --> ScanStringExpandable + NextToken --> ScanStringLiteral + NextToken --> ScanVariable + NextToken --> SkipChar + NextToken --> SkipWhiteSpace + NextToken --> UngetChar + NormalizeCRLF --> PeekChar + NormalizeCRLF --> SkipChar + ReportError --> NewScriptExtent + ReportIncompleteInput --> NewScriptExtent + ScanBlockComment --> AtEof + ScanBlockComment --> GetChar + ScanBlockComment --> NewCommentToken + ScanBlockComment --> NormalizeCRLF + ScanBlockComment --> PeekChar + ScanBlockComment --> ReportIncompleteInput + ScanBlockComment --> SkipChar + ScanBlockComment --> UngetChar + ScanDot --> InCommandMode + ScanDot --> NewToken + ScanDot --> PeekChar + ScanDot --> ScanGenericToken + ScanDot --> ScanNumber + ScanDot --> SkipChar + ScanDot --> UngetChar + ScanGenericToken --> Backtick + ScanGenericToken --> GetChar + ScanGenericToken --> GetStringBuilder + ScanGenericToken --> NewToken + ScanGenericToken --> PeekChar + ScanGenericToken --> ScanStringExpandable + ScanGenericToken --> ScanStringLiteral + ScanGenericToken --> SkipChar + ScanGenericToken --> UngetChar + ScanHereStringExpandable --> AtEof + ScanHereStringExpandable --> Backtick + ScanHereStringExpandable --> GetChar + ScanHereStringExpandable --> GetStringBuilder + ScanHereStringExpandable --> PeekChar + ScanHereStringExpandable --> ReportIncompleteInput + ScanHereStringExpandable --> SkipChar + ScanHereStringExpandable --> UngetChar + ScanHereStringLiteral --> AtEof + ScanHereStringLiteral --> GetChar + ScanHereStringLiteral --> GetStringBuilder + ScanHereStringLiteral --> PeekChar + ScanHereStringLiteral --> ReportIncompleteInput + ScanHereStringLiteral --> SkipChar + ScanHereStringLiteral --> UngetChar + ScanIdentifier --> GetChar + ScanIdentifier --> GetStringBuilder + ScanIdentifier --> InCommandMode + ScanIdentifier --> NewToken + ScanIdentifier --> ScanGenericToken + ScanIdentifier --> UngetChar + ScanLabel --> GetChar + ScanLabel --> GetStringBuilder + ScanLabel --> InCommandMode + ScanLabel --> ScanGenericToken + ScanLabel --> UngetChar + ScanLineComment --> CurrentExtent + ScanLineComment --> NewCommentToken + ScanLineComment --> ScanToEndOfCommentLine + ScanNumber --> GetStringBuilder + ScanNumber --> InExpressionMode + ScanNumber --> NewNumberToken + ScanNumber --> NewScriptExtent + ScanNumber --> ReportError + ScanNumber --> ScanGenericToken + ScanNumber --> ScanNumberHelper + ScanNumberHelper --> GetChar + ScanNumberHelper --> GetStringBuilder + ScanNumberHelper --> InExpressionMode + ScanNumberHelper --> PeekChar + ScanNumberHelper --> SkipChar + ScanNumberHelper --> UngetChar + ScanParameter --> GetChar + ScanParameter --> GetStringBuilder + ScanParameter --> InCommandMode + ScanParameter --> InExpressionMode + ScanParameter --> NewToken + ScanParameter --> ScanGenericToken + ScanParameter --> UngetChar + ScanStringExpandable --> AtEof + ScanStringExpandable --> Backtick + ScanStringExpandable --> GetChar + ScanStringExpandable --> GetStringBuilder + ScanStringExpandable --> PeekChar + ScanStringExpandable --> ReportIncompleteInput + ScanStringExpandable --> SkipChar + ScanStringExpandable --> UngetChar + ScanStringLiteral --> AtEof + ScanStringLiteral --> GetChar + ScanStringLiteral --> GetStringBuilder + ScanStringLiteral --> PeekChar + ScanStringLiteral --> ReportIncompleteInput + ScanStringLiteral --> UngetChar + ScanToEndOfCommentLine --> AtEof + ScanToEndOfCommentLine --> GetChar + ScanToEndOfCommentLine --> GetStringBuilder + ScanToEndOfCommentLine --> UngetChar + ScanUnicodeEscape --> GetChar + ScanUnicodeEscape --> GetStringBuilder + ScanUnicodeEscape --> NewScriptExtent + ScanUnicodeEscape --> ReportError + ScanUnicodeEscape --> UngetChar + ScanVariable --> AtEof + ScanVariable --> Backtick + ScanVariable --> GetChar + ScanVariable --> GetStringBuilder + ScanVariable --> InCommandMode + ScanVariable --> NewScriptExtent + ScanVariable --> NewToken + ScanVariable --> PeekChar + ScanVariable --> ReportError + ScanVariable --> ReportIncompleteInput + ScanVariable --> ScanGenericToken + ScanVariable --> UngetChar + SkipWhiteSpace --> PeekChar + SkipWhiteSpace --> SkipChar +``` + +## 4. Parse: entry chain + +From `ParseTask` through the first rules, depth 3. Nodes: 10, edges: 9. + +```mermaid +flowchart LR + ParamBlockRule --> AttributeListRule + ParamBlockRule --> ParameterListRule + ParseTask --> ScriptBlockRule + ScriptBlockBodyRule --> NamedBlockListRule + ScriptBlockBodyRule --> StatementListRule + ScriptBlockRule --> ParamBlockRule + ScriptBlockRule --> ScriptBlockBodyRule + ScriptBlockRule --> UsingStatementsRule + UsingStatementsRule --> UsingStatementRule +``` + +## 5. Check (hand-written chain) + +`Parser.ParseTask` → `ScriptBlockAst.PerformPostParseChecks` → `SymbolResolver.ResolveSymbols` (visitor over the AST) → `SemanticChecks.CheckAst` (visitor; for class methods `VariableAnalysis.AnalyzeMemberFunction`; for `data` statements `RestrictedLanguageChecker`). Compile-time: `Compiler.Compile` → `VariableAnalysis.Analyze` and, for the root block, `Compiler.GenerateTypesAndUsings` → `TypeDefiner.DefineTypes`. Evidence: [02.4-semantic-analysis.md](02.4-semantic-analysis.md) §6. + +## 6. Compile + +From `Compiler.Compile` through helpers (the 63 `Visit*` methods are omitted from the graph; see [COMPILATION-MAP.md](COMPILATION-MAP.md)), depth 3. Nodes: 3, edges: 3. + +```mermaid +flowchart LR + Compile --> AddSequencePoint + Compile --> CompileTree + CompileTree --> Compile +``` + +## 6b. Compile: script block body + +From `Compiler.VisitScriptBlock` (called through `body.Accept(this)` inside `Compile`) through the lambda builders, depth 4. Nodes: 15, edges: 14. + +```mermaid +flowchart LR + CompileNamedBlock --> CompileSingleLambda + CompileSingleLambda --> CompileStatementListWithTraps + CompileSingleLambda --> GenerateFunctionEpilog + CompileSingleLambda --> GenerateFunctionProlog + CompileSingleLambda --> GenerateTypesAndUsings + CompileStatementListWithTraps --> CompileTrap + CompileStatementListWithTraps --> CompileTrappableExpression + CompileStatementListWithTraps --> CompileTypeName + GenerateFunctionEpilog --> UpdatePosition + GenerateFunctionProlog --> AddSequencePoint + GenerateFunctionProlog --> InitializeMemberProperties + GenerateTypesAndUsings --> DefinePowerShellTypes + GenerateTypesAndUsings --> GenerateLoadUsings + VisitScriptBlock --> CompileNamedBlock +``` + +## 7. Invoke + +From `ScriptBlock.InvokeWithPipeImpl`, depth 2. Nodes: 12, edges: 14. + +```mermaid +flowchart LR + BindArgumentsForScriptblockInvoke --> SetAutomaticVariable + GetCodeToInvoke --> Compile + InvokeWithPipeImpl --> BindArgumentsForScriptblockInvoke + InvokeWithPipeImpl --> GetCodeToInvoke + InvokeWithPipeImpl --> GetFileName + InvokeWithPipeImpl --> LogScriptBlockEnd + InvokeWithPipeImpl --> LogScriptBlockStart + InvokeWithPipeImpl --> MakeLocalsTuple + InvokeWithPipeImpl --> SetAutomaticVariable + LogScriptBlockEnd --> GetScriptBlockLoggingSetting + LogScriptBlockEnd --> ToString + LogScriptBlockStart --> GetScriptBlockLoggingSetting + LogScriptBlockStart --> LogScriptBlockCreation + LogScriptBlockStart --> ToString +``` + +## 8. Parser rule-to-rule edges (complete list, `*Rule` methods reachable from `ScriptBlockRule`) + +62 rules, 124 edges (`GENERATED`, `APPROX`). Each row lists the callee rules of one rule. + +| Rule | Calls (rules) | +|---|---| +| `ArrayLiteralRule` | `UnaryExpressionRule` | +| `AttributeArgumentsRule` | `ExpressionRule`, `SimpleNameRule` | +| `AttributeListRule` | `AttributeRule` | +| `AttributeRule` | `AttributeArgumentsRule`, `TypeNameRule` | +| `BinaryExpressionRule` | `ArrayLiteralRule` | +| `BlockStatementRule` | `StatementBlockRule` | +| `BreakStatementRule` | `LabelOrKeyRule` | +| `CatchBlockRule` | `AttributeRule`, `StatementBlockRule` | +| `ClassDefinitionRule` | `ClassMemberRule`, `SimpleNameRule`, `TypeNameRule` | +| `ClassMemberRule` | `AttributeRule`, `ExpressionRule`, `MethodDeclarationRule` | +| `CommandRule` | `InlineScriptRule`, `RedirectionRule` | +| `ConfigurationStatementRule` | `ScriptBlockExpressionRule` | +| `ContinueStatementRule` | `LabelOrKeyRule` | +| `DataStatementRule` | `SimpleNameRule`, `StatementBlockRule` | +| `DoWhileStatementRule` | `PipelineChainRule`, `StatementBlockRule` | +| `DynamicKeywordStatementRule` | `CommandRule`, `HashExpressionRule`, `ScriptBlockExpressionRule` | +| `EnumDefinitionRule` | `EnumMemberRule`, `SimpleNameRule`, `TypeNameRule` | +| `EnumMemberRule` | `ExpressionRule`, `SimpleNameRule` | +| `ExitStatementRule` | `PipelineChainRule` | +| `ExpressionRule` | `BinaryExpressionRule` | +| `FinishTypeNameRule` | `GenericTypeNameRule` | +| `ForStatementRule` | `PipelineChainRule`, `StatementBlockRule` | +| `ForeachStatementRule` | `PipelineChainRule`, `StatementBlockRule` | +| `FunctionDeclarationRule` | `FunctionParameterDeclarationRule`, `ScriptBlockRule` | +| `FunctionParameterDeclarationRule` | `ParameterListRule` | +| `GenericTypeNameRule` | `GenericTypeArgumentsRule` | +| `IfStatementRule` | `PipelineChainRule`, `StatementBlockRule` | +| `InlineScriptRule` | `ScriptBlockExpressionRule` | +| `InvokeParamParenListRule` | `ExpressionRule` | +| `LabelOrKeyRule` | `SimpleNameRule`, `UnaryExpressionRule` | +| `LabeledStatementRule` | `DoWhileStatementRule`, `ForStatementRule`, `ForeachStatementRule`, `PipelineChainRule`, `SwitchStatementRule`, `WhileStatementRule` | +| `MethodDeclarationRule` | `FunctionParameterDeclarationRule`, `InvokeParamParenListRule`, `ScriptBlockRule` | +| `NamedBlockListRule` | `StatementBlockRule` | +| `ParamBlockRule` | `AttributeListRule`, `ParameterListRule` | +| `ParameterListRule` | `ParameterRule` | +| `ParameterRule` | `AttributeListRule`, `ExpressionRule` | +| `ParenthesizedExpressionRule` | `PipelineChainRule` | +| `PipelineChainRule` | `ExpressionRule`, `PipelineRule`, `StatementRule` | +| `PipelineRule` | `CommandRule`, `ExpressionRule`, `RedirectionRule` | +| `PrimaryExpressionRule` | `ExpandableStringRule`, `HashExpressionRule`, `ParenthesizedExpressionRule`, `ScriptBlockExpressionRule`, `SubExpressionRule` | +| `ReturnStatementRule` | `PipelineChainRule` | +| `ScriptBlockBodyRule` | `NamedBlockListRule`, `StatementListRule` | +| `ScriptBlockExpressionRule` | `ScriptBlockRule` | +| `ScriptBlockRule` | `ParamBlockRule`, `ScriptBlockBodyRule`, `UsingStatementsRule` | +| `StatementBlockRule` | `StatementListRule` | +| `StatementListRule` | `StatementRule` | +| `StatementRule` | `AttributeListRule`, `BlockStatementRule`, `BreakStatementRule`, `ClassDefinitionRule`, `ConfigurationStatementRule`, `ContinueStatementRule`, `DataStatementRule`, `DoWhileStatementRule`, `DynamicKeywordStatementRule`, `EnumDefinitionRule`, `ExitStatementRule`, `ForStatementRule`, `ForeachStatementRule`, `FunctionDeclarationRule`, `IfStatementRule`, `LabeledStatementRule`, `PipelineChainRule`, `ReturnStatementRule`, `SwitchStatementRule`, `ThrowStatementRule`, `TrapStatementRule`, `TryStatementRule`, `UsingStatementRule`, `WhileStatementRule` | +| `SubExpressionRule` | `StatementListRule` | +| `SwitchStatementRule` | `PipelineChainRule`, `StatementBlockRule` | +| `ThrowStatementRule` | `PipelineChainRule` | +| `TrapStatementRule` | `AttributeRule`, `StatementBlockRule` | +| `TryStatementRule` | `CatchBlockRule`, `StatementBlockRule` | +| `TypeNameRule` | `FinishTypeNameRule` | +| `UnaryExpressionRule` | `AttributeListRule`, `PrimaryExpressionRule` | +| `UsingStatementsRule` | `UsingStatementRule` | +| `WhileStatementRule` | `PipelineChainRule`, `StatementBlockRule` | + +## 9. Guidance for MicroteamShell [LESSON] + +- **LSN-CALLG-01** [REPLICATE] (area: NLD-TOOL) Keep the five entry points (tokenize, parse, check, compile, invoke) as separate callable stages with narrow signatures so each can be tested alone. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`. +- **LSN-CALLG-02** [ADAPT] (area: NLD-TOOL) Generate such graphs in CI from the shell's own sources to catch accidental coupling between stages. Why: the PowerShell graphs show the compiler calling parser-time helpers and the AST calling the compiler. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::VariableExpressionAst@L9404-L9624`. + +Verification: graphs are produced by the script from member ranges; `ParseTask`, `ScriptBlockRule`, `NextToken`, `Compiler.Compile` and `InvokeWithPipeImpl` exist in `data/members-tierA.json`; spot-checked edges: `ParseTask → ScriptBlockRule`, `NextToken → ScanVariable`, `Compile → CompileTree`. + diff --git a/docs-out/02-language/GOLDEN-SAMPLES.md b/docs-out/02-language/GOLDEN-SAMPLES.md new file mode 100644 index 0000000..0fada96 --- /dev/null +++ b/docs-out/02-language/GOLDEN-SAMPLES.md @@ -0,0 +1,97 @@ +--- +id: LANG-GOLDEN-001 +title: Golden samples +kind: guide +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [] +mirror_cards: [] +generated_from: [] +--- + +# Golden samples + +> STATIC-ONLY — **no golden samples exist** (`data/golden/` is empty by owner decision, backlog VB-001). This file is the index the study prompt requires; it states the empty state and defines the format so samples can be added when a host with `pwsh` is available. No `GS-*` identifier is used anywhere in this study. + +## 1. Purpose and scope [SPEC] + +A golden sample is a script snippet with the observable result recorded from a real `pwsh` run (token kinds, AST node types, output objects, error ids, `$?`). Samples turn `UNVERIFIED` behavior claims into verified ones and feed the conformance corpus of the new shell. + +## 2. Source map + +Not applicable (no data). + +## 3. Responsibilities and non-responsibilities + +Index and format only. Execution of samples is specified in [ORACLE-PROTOCOL.md](ORACLE-PROTOCOL.md). + +## 4. Key types and contracts [IMPL] + +Planned record (JSON, one file per sample under `data/golden/`): `id` (`GS--`, assigned only when a sample is created from a run), `snippet`, `pwsh_version`, `commit`, `tokens`, `ast_node_types`, `output` (typed, serialized), `errors` (ids and messages), `question_mark`, `streams`, `notes`. + +## 5. Data model + +Coverage report (computed when samples exist): per document claim marked `UNVERIFIED`, whether a sample covers it. Today: 0 samples, coverage 0%; every behavior statement in the study is `FACT-SRC` (read) or `INFERENCE`/`UNVERIFIED`. + +## 6. Control flow + +Not applicable. + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +Not applicable: no samples. + +## 9. Error handling and edge cases + +A sample whose run depends on the host (culture, OS) must record those inputs. + +## 10. Concurrency and threading + +Not applicable. + +## 11. Extension points and customization hooks + +Samples are added by the oracle harness; claims are re-labelled by `tools/consist.py check-claims` once a sample id is attached. + +## 12. Dependencies + +[ORACLE-PROTOCOL.md](ORACLE-PROTOCOL.md), [SEMANTIC-QUIRKS.md](SEMANTIC-QUIRKS.md), [REFERENCE-TEST-CORPUS.md](REFERENCE-TEST-CORPUS.md). + +## 13. Performance notes + +Not applicable. + +## 14. Security considerations (mandatory) + +Running snippets executes code; the protocol requires a scratch copy and no network. No security surface in this index. + +## 15. Enterprise considerations + +Not applicable. + +## 16. Tests + +The existing repository tests are the interim behavior evidence ([REFERENCE-TEST-CORPUS.md](REFERENCE-TEST-CORPUS.md)). + +## 17. Design decisions and trade-offs + +Owner decision: static-only run (D-001..D-011, VB-001). + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-GOLD-01** [REPLICATE] (area: NLD-TOOL) Keep an oracle-recorded conformance corpus as the acceptance test of the new interpreter. Evidence: `test/powershell/Language/Parser/` (the repository's own language tests, see REFERENCE-TEST-CORPUS). + +## 19. Open questions / UNVERIFIED + +All samples are future work (VB-001). + +Verification: `data/golden/` was found empty at the start of the run. diff --git a/docs-out/02-language/INTERPRETER-DESIGN-SPACE.md b/docs-out/02-language/INTERPRETER-DESIGN-SPACE.md new file mode 100644 index 0000000..2383f98 --- /dev/null +++ b/docs-out/02-language/INTERPRETER-DESIGN-SPACE.md @@ -0,0 +1,237 @@ +--- +id: LANG-DESIGNSPACE-001 +title: Interpreter design space +kind: guide +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/interpreter/LightCompiler.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/interpreter/_folder.md] +generated_from: [data/interpreter-scores.json] +--- + +# Interpreter design space + +> STATIC-ONLY — behavior not run-verified. Facts about PowerShell's own strategy (`S0`) are **linked, not restated**: [TIERING-POLICY.md](TIERING-POLICY.md), [IR-SPEC.md](IR-SPEC.md), [INSTRUCTION-SET.md](INSTRUCTION-SET.md), [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). `FACT-SRC` = repository evidence, `EXTERNAL` = .NET facts (cross-check only), `INFERENCE` = this study's reasoning. A claim that PowerShell does not interpret the AST directly is a statement about PowerShell (`S0`); a recommendation to start with an AST tree-walker is a statement about the new shell (`S1`); they are kept separate (contradiction candidate C5 of the study prompt). + +## 1. Purpose and scope [SPEC] + +Choose an execution strategy for MicroteamShell (C#/.NET, personal use, Windows). This document describes the options, scores them with fixed criteria and weights, applies hard gates, and gives two recommendations (JIT allowed, NativeAOT required) with sensitivity and a build order. + +## 2. Anatomy of PowerShell's strategy (links only) + +What is compiled, interpreted and dynamic: AST → expression trees ([COMPILATION-MAP.md](COMPILATION-MAP.md)); trees → instruction array or JIT delegate ([TIERING-POLICY.md](TIERING-POLICY.md) §7.2); dynamic call sites with per-site caches ([BINDER-CATALOG.md](BINDER-CATALOG.md) §4); start-up and warm-up (interpret first, JIT after counters, optimized variant only without debugging, [TIERING-POLICY.md](TIERING-POLICY.md) §7.1); caching (script block cache, shared compiled data, [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §4). Equivalence obligations between tiers: [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). Facts versus inference are separated in those documents. + +## 3. .NET constraints (EXTERNAL, cross-check only) + +- NativeAOT forbids `Reflection.Emit` and runtime code generation; `Expression.Compile` then falls back to an interpreter with degraded performance (`EXTERNAL`). +- Lightweight code generation used by the DLR is unavailable under AOT (`EXTERNAL`). +- `AssemblyLoadContext` allows collectible assemblies, which bound the memory of dynamic assemblies, but Windows-only personal use does not need unloading (`EXTERNAL`, `INFERENCE`). +- ReadyToRun reduces JIT start-up cost at the price of larger binaries and less optimized code until tiered JIT replaces it (`EXTERNAL`). + +## 4. Options + +`S0` PowerShell-faithful tiered pipeline; `S1` AST tree-walking interpreter; `S2` custom bytecode VM; `S3` `System.Linq.Expressions` compilation to delegates (cache and invalidation by script block identity); `S4` `Reflection.Emit` dynamic assemblies; `S5` source-generated ahead-of-time code for built-in commands (a complement, not a script engine); `S6` hybrid of `S1` and `S2` with count-based promotion. S2 sketch: a stack machine over an `object` stack with locals in a frame array, one instruction per AST operation, jump offsets for loops and `try` regions, explicit continuation records for `finally` (the structure of the instruction set in [INSTRUCTION-SET.md](INSTRUCTION-SET.md)). + +## 5. Scoring (rendered from data) + +Weights sum to 100 (fidelity 25, sandboxability 15, startup 10, throughput 10, debuggability 10, AOT-friendliness 10, determinism 5, memory 5, observability 5, simplicity 5). Scores 1-5 from `data/interpreter-scores.json`; totals, gates and recommendations are computed by `tools/gen_coord.py` (7 options). + + +| Option | Name | Behavioral fidelity (25) | Sandboxability (15) | Startup latency (10) | Steady-state throughput (10) | Debuggability (10) | AOT-friendliness (10) | Determinism (5) | Memory footprint (5) | Observability (5) | Implementation simplicity (5) | Weighted total (max 5) | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| S0 | PowerShell-faithful tiered pipeline (AST → expression trees → light interpreter → JIT) | 5 | 3 | 3 | 5 | 4 | 1 | 2 | 2 | 4 | 1 | **3.45** | +| S1 | AST tree-walking interpreter | 4 | 5 | 5 | 2 | 5 | 5 | 5 | 4 | 4 | 5 | **4.35** | +| S2 | Custom bytecode VM | 4 | 5 | 4 | 3 | 3 | 5 | 5 | 4 | 3 | 2 | **3.95** | +| S3 | System.Linq.Expressions compilation to delegates | 5 | 3 | 2 | 5 | 3 | 2 | 3 | 2 | 3 | 3 | **3.45** | +| S4 | Reflection.Emit dynamic assemblies | 3 | 2 | 1 | 5 | 2 | 1 | 3 | 1 | 2 | 1 | **2.30** | +| S5 | Source-generated ahead-of-time compilation (built-in commands) | 2 | 4 | 5 | 5 | 2 | 5 | 5 | 5 | 3 | 3 | **3.60** | +| S6 | Hybrid: tree-walker plus bytecode tier for hot paths | 4 | 5 | 4 | 4 | 3 | 5 | 4 | 3 | 3 | 2 | **3.95** | + +Justifications (one sentence per score, label in brackets): + +**S0 PowerShell-faithful tiered pipeline (AST → expression trees → light interpreter → JIT)** +- Behavioral fidelity = 5 [FACT-SRC] It is the strategy PowerShell itself uses, so observable behavior is reproduced by construction (TIERING-POLICY, EQUIVALENCE-SPEC). +- Sandboxability = 3 [INFERENCE] Expression trees and call sites run arbitrary delegates; hooks exist at binders and runtime helpers but the JIT tier bypasses interpreter-level checks. +- Startup latency = 3 [FACT-SRC] The interpreter tier starts cheaply but the compiler front end builds trees for every block (COMPILATION-MAP). +- Steady-state throughput = 5 [FACT-SRC] Hot blocks and loops are JIT-compiled after 32 calls or 16 iterations (TIERING-POLICY). +- Debuggability = 4 [FACT-SRC] Sequence points and debugger hooks are emitted in both tiers (IR-SPEC §5). +- AOT-friendliness = 1 [EXTERNAL] Expression.Compile and Reflection.Emit need a JIT; under NativeAOT trees fall back to a slow interpreter. +- Determinism = 2 [FACT-SRC] Asynchronous promotion makes the executing tier nondeterministic (TIERING-POLICY §7.5). +- Memory footprint = 2 [FACT-SRC] Trees, instruction arrays, call sites and caches are retained per block. +- Observability = 4 [FACT-SRC] ETW compile events and script block logging exist (02.5 §14). +- Implementation simplicity = 1 [FACT-SRC] About 6,600 lines of compiler plus the interpreter, binders and DLR dependency. + +**S1 AST tree-walking interpreter** +- Behavioral fidelity = 4 [INFERENCE] Every protocol of IR-SPEC §5 can be implemented once in the evaluator; no PowerShell-visible behavior requires a compiled form. +- Sandboxability = 5 [INFERENCE] All execution passes through one evaluator, so policy checks sit at every node visit and call. +- Startup latency = 5 [INFERENCE] No compile step: execution starts at the first node. +- Steady-state throughput = 2 [INFERENCE] Each node costs a virtual dispatch and allocations for values; no slot access without a resolver pass. +- Debuggability = 5 [INFERENCE] The evaluator is the debugger: breakpoints and stepping are plain hooks on statements. +- AOT-friendliness = 5 [EXTERNAL] Pure managed code without dynamic code generation runs under NativeAOT. +- Determinism = 5 [INFERENCE] Single tier: results never depend on promotion timing. +- Memory footprint = 4 [INFERENCE] Only the AST is retained; no trees or instruction arrays. +- Observability = 4 [INFERENCE] Trace of statements and calls is trivial to emit. +- Implementation simplicity = 5 [INFERENCE] Smallest implementation; the main work is the language rules, not an engine. + +**S2 Custom bytecode VM** +- Behavioral fidelity = 4 [INFERENCE] A VM can implement the same protocols; the extra translation step adds risk of divergence from the AST semantics. +- Sandboxability = 5 [INFERENCE] A VM executes only its own instructions, so hooks go into the instruction loop. +- Startup latency = 4 [INFERENCE] A compile step to bytecode is cheap compared with trees but not free. +- Steady-state throughput = 3 [INFERENCE] A tight instruction loop beats tree-walking (PowerShell's own interpreter is of this kind, INSTRUCTION-SET). +- Debuggability = 3 [INFERENCE] Needs source maps from instructions to sequence points. +- AOT-friendliness = 5 [EXTERNAL] Managed bytecode interpreter has no AOT constraints. +- Determinism = 5 [INFERENCE] Single tier, deterministic. +- Memory footprint = 4 [INFERENCE] Instruction arrays and constant pools per block. +- Observability = 3 [INFERENCE] Instruction-level tracing is possible but noisy. +- Implementation simplicity = 2 [INFERENCE] A compiler, an instruction set and a frame model must be designed and tested. + +**S3 System.Linq.Expressions compilation to delegates** +- Behavioral fidelity = 5 [FACT-SRC] PowerShell compiles to expression trees; behavior is reproduced via the same IR (COMPILATION-MAP). +- Sandboxability = 3 [INFERENCE] Compiled delegates run unchecked CLR code once produced; policy must be emitted into the trees. +- Startup latency = 2 [FACT-SRC] Tree building and Expression.Compile cost start-up time; PowerShell avoids it by interpreting first. +- Steady-state throughput = 5 [FACT-SRC] JIT-compiled delegates are the fast path of PowerShell hot code. +- Debuggability = 3 [INFERENCE] Debug info is possible (DebugInfo nodes) but stepping compiled code is harder. +- AOT-friendliness = 2 [EXTERNAL] Expression.Compile under NativeAOT degrades to an interpreter. +- Determinism = 3 [INFERENCE] Compile on demand with caching is deterministic if synchronous. +- Memory footprint = 2 [FACT-SRC] Delegates and trees are cached per block (CompiledScriptBlockData). +- Observability = 3 [INFERENCE] Compile events can be logged. +- Implementation simplicity = 3 [FACT-SRC] Needs the compiler, binders and caches but not the light interpreter. + +**S4 Reflection.Emit dynamic assemblies** +- Behavioral fidelity = 3 [FACT-SRC] PowerShell uses Reflection.Emit only to define classes and enums (TypeDefiner), not for the dynamic language core. +- Sandboxability = 2 [INFERENCE] Emitted code calls the CLR directly; checks must be emitted explicitly. +- Startup latency = 1 [EXTERNAL] Emitting and loading assemblies costs the most start-up time of the options. +- Steady-state throughput = 5 [EXTERNAL] Emitted IL is JIT-compiled to native code. +- Debuggability = 2 [INFERENCE] Debugging emitted code needs symbol emission. +- AOT-friendliness = 1 [EXTERNAL] Reflection.Emit is unavailable under NativeAOT. +- Determinism = 3 [INFERENCE] Deterministic if emission is synchronous. +- Memory footprint = 1 [EXTERNAL] Dynamic assemblies stay loaded unless collectible (AssemblyLoadContext). +- Observability = 2 [INFERENCE] Emitted code is opaque to tracing. +- Implementation simplicity = 1 [INFERENCE] Emitting IL for dynamic scoping, pipelines and error-action semantics is the most complex option. + +**S5 Source-generated ahead-of-time compilation (built-in commands)** +- Behavioral fidelity = 2 [INFERENCE] It cannot run user scripts; it only compiles the shell's own built-in commands. +- Sandboxability = 4 [INFERENCE] Generated code is reviewed at build time and calls only allowed APIs. +- Startup latency = 5 [EXTERNAL] AOT or R2R code needs no JIT at start-up. +- Steady-state throughput = 5 [EXTERNAL] Native code is the fastest. +- Debuggability = 2 [INFERENCE] Generated code is debuggable like normal C#. +- AOT-friendliness = 5 [EXTERNAL] Fully compatible with NativeAOT. +- Determinism = 5 [INFERENCE] Deterministic. +- Memory footprint = 5 [EXTERNAL] No runtime code generation memory. +- Observability = 3 [INFERENCE] Generated hooks can add tracing. +- Implementation simplicity = 3 [INFERENCE] A source generator is extra work but each command stays plain code. + +**S6 Hybrid: tree-walker plus bytecode tier for hot paths** +- Behavioral fidelity = 4 [INFERENCE] Behavior is defined by the tree-walker; the bytecode tier must match it (EQUIVALENCE-SPEC applies). +- Sandboxability = 5 [INFERENCE] Both tiers are interpreters, so hooks are placed in two loops. +- Startup latency = 4 [INFERENCE] Starts as a tree-walker; bytecode is generated only for hot code. +- Steady-state throughput = 4 [INFERENCE] Hot paths run the bytecode loop. +- Debuggability = 3 [INFERENCE] Two tiers need two debug paths. +- AOT-friendliness = 5 [EXTERNAL] Managed code only; no JIT requirement. +- Determinism = 4 [INFERENCE] Deterministic if promotion is synchronous and count-based. +- Memory footprint = 3 [INFERENCE] AST plus bytecode for hot blocks. +- Observability = 3 [INFERENCE] Promotion events can be traced. +- Implementation simplicity = 2 [INFERENCE] Two engines and an equivalence test harness. + +Hard gates (H1 reproduces `CORE` observable behavior; H2 feasible under the deployment constraint; H3 policy hooks possible at every boundary of [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md)): + +| Option | H1 | H2 JIT-allowed | H2 NativeAOT-required | H3 | Gate note | +|---|---|---|---|---|---| +| S0 | pass | pass | FAIL | pass | AOT: JIT tier and Expression.Compile unavailable (EXTERNAL) | +| S1 | pass | pass | pass | pass | Fidelity depends on implementing the protocols of IR-SPEC §5 and the equivalence obligations EQ-01..EQ-08 | +| S2 | pass | pass | pass | pass | Must reproduce flow-control and error-action protocols in the instruction set | +| S3 | pass | pass | FAIL | pass | AOT: Expression.Compile falls back to interpretation (EXTERNAL) | +| S4 | FAIL | pass | FAIL | FAIL | No evidence it can host the dynamic-scoping core (INFERENCE); emitted code bypasses evaluator hooks | +| S5 | FAIL | pass | pass | pass | Complement for built-in commands, not a strategy for user scripts | +| S6 | pass | pass | pass | pass | Build only after S1 is verified; triggers in INTERPRETER-DESIGN-SPACE §7 | + +Recommendations (computed; label `INFERENCE`): + +- **JIT-allowed**: S1 AST tree-walking interpreter (4.35); runner-up S2 Custom bytecode VM (3.95); candidates passing all gates: S1, S2, S6, S3, S0. +- **NativeAOT-required**: S1 AST tree-walking interpreter (4.35); runner-up S2 Custom bytecode VM (3.95); candidates passing all gates: S1, S2, S6. + +Sensitivity (weight of one criterion raised, other weights scaled to keep the sum at 100; smallest weight at which the JIT-allowed winner changes): + +- Behavioral fidelity (weight 25): winner becomes S3 at weight 61 +- Sandboxability (weight 15): no change up to weight 70 +- Startup latency (weight 10): no change up to weight 70 +- Steady-state throughput (weight 10): winner becomes S6 at weight 26 +- Debuggability (weight 10): no change up to weight 70 +- AOT-friendliness (weight 10): no change up to weight 70 +- Determinism (weight 5): no change up to weight 70 +- Memory footprint (weight 5): no change up to weight 70 +- Observability (weight 5): no change up to weight 70 +- Implementation simplicity (weight 5): no change up to weight 70 + + +## 6. Reading the result + +`INFERENCE`. The fidelity criterion rewards options whose behavior is defined by one evaluator: with the protocols of [IR-SPEC.md](IR-SPEC.md) §5 as the contract, `S1` can satisfy the equivalence obligations EQ-01 to EQ-08 trivially (single tier), which is why it passes H1 despite not copying PowerShell's architecture. Throughput is where `S1` loses; `S6` is the first step if measurements demand it. The recommendation is the same under both deployment constraints because the only options that fail the NativeAOT gate (`S0`, `S3`, `S4`) were already behind `S1` and `S6` on the weighted total. This is a property of the weights; the sensitivity list above shows which weight change flips the JIT-allowed winner. + +## 7. Build order with triggers + +1. **Phase A — `S1` tree-walker**, implementing the protocols of [IR-SPEC.md](IR-SPEC.md) §5 and the conformance corpus; verified against the oracle ([ORACLE-PROTOCOL.md](ORACLE-PROTOCOL.md)) before any optimizer exists. Trigger to leave Phase A: the benchmark set (tight loops of 1e6 iterations, function-call heavy recursion depth 1e4, pipeline of 1e5 objects) exceeds the owner's acceptable time (owner sets the numbers; suggested starting threshold: any benchmark above 5 s). +2. **Phase B — measured optimizations inside `S1`**: variable slots resolved by a pre-pass ([02.4-semantic-analysis.md](02.4-semantic-analysis.md) §7.2 simplified), cached operator dispatch, constant folding. Trigger to leave: benchmarks still above threshold after B, or a profile shows more than 50% of time in node dispatch. +3. **Phase C — `S6`**: bytecode tier for hot loops and functions with synchronous count-based promotion and a differential test harness (every corpus snippet run in both tiers, [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md) §7.2). +4. **Phase D — `S5`** for built-in commands only if start-up of the built-in command set is the measured bottleneck. + +## 8. Observable behavior [SPEC] + +Not applicable (design document). + +## 9. Error handling and edge cases + +Hard-gate failures are reported per option in the gate table; ties go to the simpler option. + +## 10. Concurrency and threading + +Phase C promotion must be synchronous (determinism); no background compilation. + +## 11. Extension points and customization hooks + +The `Executable` contract of B4 in [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md) lets Phase C replace the executable without touching the front end. + +## 12. Dependencies + +[STAGE-CONTRACTS.md](STAGE-CONTRACTS.md), [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md), [TIERING-POLICY.md](TIERING-POLICY.md). + +## 13. Performance notes + +Scores for throughput and start-up are `INFERENCE` from structure, not measurements. + +## 14. Security considerations (mandatory) + +Single-evaluator strategies (`S1`, `S2`, `S6`) give one place for policy checks (gate H3); compiled strategies need checks emitted into generated code. + +## 15. Enterprise considerations + +Debuggability and observability criteria favor interpreters. + +## 16. Tests + +The first executable component is the oracle-diffed tree-walker. + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| recommend `S1` first under both constraints | `INFERENCE` | least code, most debuggable; revisit with measurements | +| keep promotion synchronous and count-based | `INFERENCE` | determinism over peak performance | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-DESIGN-01** [ADAPT] (area: NLD-RUN) Build `S1`, a tree-walking evaluator that implements the protocols of IR-SPEC §5, as the first and only engine until measurements justify another tier. Why: same observable behavior as PowerShell's compiled pipeline with a fraction of the machinery. Evidence: `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661`. +- **LSN-DESIGN-02** [REPLICATE] (area: NLD-RUN) If a second tier is added, test it differentially against the first on the whole corpus. Evidence: `src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda@L43-L303`. +- **LSN-DESIGN-03** [AVOID] (area: NLD-RUN) Do not adopt `Reflection.Emit` or `Expression.Compile` as the core engine. Evidence: `src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505`. DEV: DEV-DESIGN-110 + +## 19. Open questions / UNVERIFIED + +- Scores are structural estimates; no benchmark exists. +- The owner's performance thresholds are not set. + +Verification: weights and scores live in `data/interpreter-scores.json`; totals, gates, recommendations and sensitivity are computed (`tools/gen_coord.py`); S0 facts are links to documents that cite source. diff --git a/docs-out/02-language/LANGUAGE-SPEC-DRAFT.md b/docs-out/02-language/LANGUAGE-SPEC-DRAFT.md new file mode 100644 index 0000000..dc33189 --- /dev/null +++ b/docs-out/02-language/LANGUAGE-SPEC-DRAFT.md @@ -0,0 +1,127 @@ +--- +id: LANG-SPEC-001 +title: Language specification draft +kind: guide +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/LanguagePrimitives.cs] +mirror_cards: [] +generated_from: [] +--- + +# Language specification draft + +> STATIC-ONLY — baseline behavior specification of the target language (= PowerShell 7.6.9's observable behavior, `CORE` scope), assembled from the study documents. Every statement links to the section that holds its evidence; there are no golden samples, so confirmation with `pwsh` is open (VB-001). All sections are `SPEC`. Each major section ends with a `DEVIATIONS:` note listing the proposals (`DEV-*`) that concern it, or "none"; canvas rows are in [new-language-design-canvas.md](../09-synthesis/new-language-design-canvas.md). + +## 1. Purpose and scope [SPEC] + +What a MicroteamShell implementation must do to be PowerShell-faithful at the language core: lexical structure, grammar summary, types and conversions, expressions, statements, scoping, pipeline semantics, error semantics. Out of scope: cmdlets, providers, modules, remoting, DSC, formatting. + +## 2. Source map + +Evidence documents: [02.1](02.1-tokenizer.md), [02.2](02.2-parser.md), [GRAMMAR](GRAMMAR.ebnf.md), [02.3](02.3-ast.md), [02.4](02.4-semantic-analysis.md), [02.5](02.5-compiler-and-engine.md), [02.6](02.6-types-and-operators.md), [OPERATOR-TABLE](OPERATOR-TABLE.md), [CONVERSION-RULES](CONVERSION-RULES.md), [SCOPE](SCOPE-AND-VARIABLES.md), [RTOPS](RUNTIME-OPERATIONS.md), [SB](SCRIPT-BLOCKS.md), [QUIRKS](SEMANTIC-QUIRKS.md). + +## 3. Lexical structure + +1. A script is Unicode text; lines end with `CR`, `LF` or `CRLF` ([02.1 §7.6](02.1-tokenizer.md)). Newline is a token, not whitespace; space, tab, form feed, vertical tab, no-break space and next-line are whitespace. +2. The tokenizer works in four modes — command, expression, type name, signature — chosen by the parser; text is tokenized lazily ([02.1 §6.2](02.1-tokenizer.md), CF-LEX-001). +3. **Comments**: `# …` to end of line; `<# … #>` not nested; `#requires` lines are pragmas ([02.1 §7.4](02.1-tokenizer.md)). +4. **Line continuation**: a backtick before a newline; whitespace after the backtick is skipped; a newline inside `()`, after a binary operator or after `|`, `&&`, `||`, `,` also continues ([02.2 §6.3](02.2-parser.md), CF-GRM-001). +5. **Identifiers and keywords**: keywords (51 in the keyword table, [TOKEN-CATALOG §5](TOKEN-CATALOG.md)) are case-insensitive and recognized only in command (and signature) mode; elsewhere they are identifiers or barewords. +6. **Variables**: `$name`, `${any text}`, `$scope:name`, `$drive:name`, `@name` (splatting); `$?`, `$^`, `$$`, `$_` are single-character names ([02.1 §7.3](02.1-tokenizer.md)). +7. **Numbers**: decimal, hex `0x`, binary `0b`, real with exponent; type suffixes `u s l d y n` (and `ul us uy`); multipliers `kb mb gb tb pb` (1024 powers); the suffix comes before the multiplier; a number must end at a token boundary (CF-LEX-002). +8. **Strings**: single-quoted (`''` escapes a quote, no backtick escapes), double-quoted (backtick escapes `` `0 `a `b `e `f `n `r `t `v `u{…} ``, `""` escapes a quote, `$var` / `${…}` / `$(…)` expand), here-strings `@'…'@` / `@"…"@` with the footer at the start of a line; smart quotes and en/em dashes are accepted as quotes and dashes ([02.1 §7.2](02.1-tokenizer.md), [§5](02.1-tokenizer.md)). +9. **Operators and punctuation**: symbol operators and dash operators (`-eq`, `-and`, …) — the latter only in expression mode ([OPERATOR-TABLE](OPERATOR-TABLE.md)); redirection tokens `>`, `>>`, `2>`, `2>&1`, `*>` (streams 1-6 and all) ([02.1 §7.5](02.1-tokenizer.md)). + +DEVIATIONS: DEV-LEX-001 (no `#requires`/signature handling in the tokenizer); canvas row NLD-LEX. + +## 4. Grammar summary + +The normative grammar is [GRAMMAR.ebnf.md](GRAMMAR.ebnf.md). Summary: a script block is `using` statements, an optional `param` block, then either named blocks (`begin process end dynamicparam clean`) or a statement list. Statements are separated by optional newlines and semicolons; no terminator is needed after a block statement (CF-GRM-001). A pipeline is expressions or commands joined by `|`; pipelines chain with `&&`/`||` and can run in the background with `&`. Assignment has a statement on its right. Binary operators are left-associative with the levels of CF-OP-001; the comma binds tighter than binary operators; ternary and assignment bind looser. Hashtable values are statements. Parse errors are collected, never thrown for bad input, and a tree is still produced (CF-ERR-002). + +DEVIATIONS: DEV-GRM-010 (parsing without side effects); canvas row NLD-GRM. + +## 5. Types and conversions + +1. Values are .NET objects; `$null` is null; "no value" (`AutomationNull`) is distinct; wrappers carry extra members ([02.6 §4](02.6-types-and-operators.md)). +2. Conversion follows the ordered search of CF-CNV-001 with ranks; strings convert to numbers with the literal syntax; numbers to strings with invariant formatting; `IsTrue` defines truthiness ([CONVERSION-RULES](CONVERSION-RULES.md)). +3. Arithmetic promotes to the wider type, overflows `int` to `double`, and divides `int/int` exactly or as `double` ([02.6 §7.2](02.6-types-and-operators.md)). +4. Collections are enumerated per CF-PIP-001; arrays are `object[]` unless typed; hashtables are case-insensitive (`Hashtable`/`OrderedDictionary`). + +DEVIATIONS: DEV-TYP-070 (no language-mode gate in conversion); canvas row NLD-TYP. + +## 6. Expressions + +Operators and their per-type behavior: [OPERATOR-TABLE §4](OPERATOR-TABLE.md). Member access `.` and `::`, method calls (no whitespace before `(`), indexing and slicing, casts, array and hashtable literals, sub-expressions `$()`/`@()`, script block literals, ternary and null-coalescing, `-f`, ranges ([GRAMMAR §7](GRAMMAR.ebnf.md), [BINDER-CATALOG](BINDER-CATALOG.md)). Constant expressions are folded only for foldable operators ([02.3 §7.1](02.3-ast.md)). + +DEVIATIONS: DEV-BIND-050 (no COM/ETS-adapter member binding); canvas row NLD-OP. + +## 7. Statements + +`if/elseif/else`, `while`, `do … while|until`, `for`, `foreach`, `switch` (all matching clauses run; flags regex/wildcard/exact/casesensitive/file), `break`/`continue` with labels, `return`, `exit`, `throw`, `try/catch/finally`, `trap`, `function`/`filter` with `param`, `begin/process/end`, `data`, `class`/`enum`/`using` (EXTENDED) ([GRAMMAR §4-5](GRAMMAR.ebnf.md), [02.5 §6.3-6.4](02.5-compiler-and-engine.md)). Semantic rules (duplicate parameters, loop-control placement, assignment targets, constant attribute arguments) are errors at parse time ([02.4 §8](02.4-semantic-analysis.md)). + +DEVIATIONS: DEV-FEAT-091 (reject workflow keywords at parse time); canvas row NLD-GRM. + +## 8. Scoping and variables + +CF-SCP-001: dynamic-scope reads, local writes, qualifiers `global: script: local: private:`, options `ReadOnly Constant Private AllScope`, new scope per function/script block call, dot-sourcing without scope, `$using:` for remote-style blocks, closures by snapshot ([SCOPE](SCOPE-AND-VARIABLES.md), [SB](SCRIPT-BLOCKS.md)). Strict mode by version walks scopes ([execution context](../03-runtime/execution-context-and-session-state.md) §4). + +DEVIATIONS: DEV-SCP-080, DEV-CTX-081 (no language-mode trust marking); canvas row NLD-SCP. + +## 9. Pipeline semantics + +A statement's value is output only for bare expressions and commands; output is written to the current pipe and unrolled one level (CF-PIP-001); `return` writes output then leaves; conditions, assignments, `$()` and `@()` capture output in a temporary pipe; commands run `begin`/`process`/`end` with `$_` and `$input` ([02.5 §6.5](02.5-compiler-and-engine.md), [LIFECYCLE](LIFECYCLE-script-execution.md)). Pipeline chains `&&`/`||` test `$?`. + +DEVIATIONS: none (canvas row NLD-PIP). + +## 10. Error semantics + +Three channels: parse diagnostics (before running), runtime exceptions, error records on the error stream ([STAGE-CONTRACTS §9](STAGE-CONTRACTS.md)). Each statement is guarded: an error is recorded (`$Error`, `$?` false), then `$ErrorActionPreference` and traps decide to continue, stop or ignore; `throw` terminates unless trapped or caught; `try/catch/finally` matches typed handlers in order and sets `$_` to the error record ([RTOPS §6.1](RUNTIME-OPERATIONS.md), [02.5 §6.4](02.5-compiler-and-engine.md)). Flow-control signals pass through all handlers. + +DEVIATIONS: none (CF-ERR-001 is documented with the errors document in Phase 3); canvas row NLD-ERR. + +## 11. Observable-behavior quirks + +See [SEMANTIC-QUIRKS.md](SEMANTIC-QUIRKS.md) (32 quirks with verdicts). + +DEVIATIONS: DEV-LEX-001, DEV-GRM-010, DEV-SEM-030. + +## 12. Dependencies + +Documents listed in §2. + +## 13. Performance notes + +Not applicable (behavior only); execution strategy: [INTERPRETER-DESIGN-SPACE.md](INTERPRETER-DESIGN-SPACE.md). + +## 14. Security considerations (mandatory) + +Language modes, restricted data sections, constrained conversions and parse-time side effects are described in the linked sections; the baseline includes restricted-language checking for `data` sections ([02.4 §14](02.4-semantic-analysis.md)). + +## 15. Enterprise considerations + +Diagnostics, logging and positions: [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md) §15. + +## 16. Tests + +[REFERENCE-TEST-CORPUS.md](REFERENCE-TEST-CORPUS.md). + +## 17. Design decisions and trade-offs + +The specification is behavior-first; mechanisms (DLR, expression trees, binders) are not part of it. + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-SPEC-01** [REPLICATE] (area: NLD-GRM) Treat this specification plus the quirks catalog as the acceptance baseline and add oracle-confirmed tests per statement. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8191`. +- **LSN-SPEC-02** [ADAPT] (area: NLD-TOOL) Keep the specification in the shell's repository as executable examples (snippet + expected result) rather than prose only. Why: same content, checkable. Evidence: `test/powershell/Language/Parser/` (test style). + +## 19. Open questions / UNVERIFIED + +All statements are `FACT-SRC` or `INFERENCE` per the linked sections; the oracle (VB-001) is needed to promote them. The command invocation and parameter binding rules (Phase 3) are not part of this draft. + +Verification: each paragraph cites the section holding its evidence; counts (keywords, quirks) come from registered numbers. diff --git a/docs-out/02-language/LIFECYCLE-script-execution.md b/docs-out/02-language/LIFECYCLE-script-execution.md new file mode 100644 index 0000000..639d642 --- /dev/null +++ b/docs-out/02-language/LIFECYCLE-script-execution.md @@ -0,0 +1,158 @@ +--- +id: LANG-LIFECYCLE-001 +title: Script execution lifecycle +kind: guide +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/Microsoft.PowerShell.ConsoleHost/host/msh/ManagedEntrance.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs, src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs, src/System.Management.Automation/engine/hostifaces/Command.cs, src/System.Management.Automation/engine/AutomationEngine.cs, src/System.Management.Automation/engine/ScriptCommandProcessor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/hostifaces/_folder.md] +generated_from: [] +--- + +# Script execution lifecycle + +> STATIC-ONLY — behavior not run-verified. End-to-end sequence from `pwsh -c ""` to output, naming the classes and methods touched. Host steps are summarized from [layered-architecture](../00-overview/layered-architecture.md); the engine steps were read for this document (`FACT-SRC`); steps whose bodies were not read are marked `UNVERIFIED`. Stage contracts: [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md); call graphs: [CALL-GRAPHS.md](CALL-GRAPHS.md). + +## 1. Purpose and scope [SPEC] + +One continuous trace of what happens to the text of a command line script, with the decision points that matter to a re-implementation: where parsing happens, where the language-mode and restricted-language checks run, where the script block is compiled, how the tier is chosen, and how output reaches the host. + +## 2. Source map + +| Step group | Path | Type | Role | +|---|---|---|---| +| Entry | `src/powershell/Program.cs`, `ManagedEntrance.cs` | `ManagedPSEntry`, `UnmanagedPSEntry` | process entry and start-up | +| Host | `ConsoleHost.cs`, `Executor.cs` | `ConsoleHost`, `Executor` | command-line handling, runspace loop, run a command string | +| Pipeline | src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline@L21-L1143, src/System.Management.Automation/engine/hostifaces/Command.cs::Command@L18-L774 | | pipeline of commands, command processor creation | +| Parse | src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine@L15-L128 | | `ParseScriptBlock` | +| Run | src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor@L232-L661 | | runs a script block as a command | + +## 3. Responsibilities and non-responsibilities + +Lifecycle only; stage internals are in the stage documents. + +## 4. Key types and contracts [IMPL] + +Contracts per boundary: [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md) §4. + +## 5. Data model + +Objects created along the way: `Runspace` + `ExecutionContext`, `Pipeline` with one `Command` (`IsScript`), `PipelineProcessor`, `ScriptBlock`/`ScriptBlockAst`, `DlrScriptCommandProcessor` with `FunctionContext` and locals tuple, the output `Pipe`, error records. + +## 6. Control flow + +```mermaid +sequenceDiagram + participant M as ManagedPSEntry.Main + participant U as UnmanagedPSEntry.Start + participant H as ConsoleHost + participant X as Executor + participant L as LocalPipeline + participant C as Command.CreateCommandProcessor + participant E as AutomationEngine.ParseScriptBlock + participant P as Parser + participant D as DlrScriptCommandProcessor + participant S as ScriptBlock and Compiler + participant R as LightLambda and Interpreter + M->>U: Start(args) + U->>H: ParseCommandLine, ConsoleHost.Start + H->>H: DoRunspaceLoop, CreateRunspace + H->>X: ExecuteCommand(initial command) + X->>L: CreatePipeline, ExecuteCommandHelper (adds Out-Default) + L->>L: InvokeHelper, CreatePipelineProcessor + L->>C: for each Command + C->>E: ParseScriptBlock(text, addToHistory) + E->>P: EngineParser.Parse(text, ParseMode.Default) + P-->>E: ScriptBlockAst + ParseError[] + E-->>C: new ScriptBlock(ast, isFilter false) or ParseException + C->>D: new DlrScriptCommandProcessor(scriptBlock, ...) + D->>S: Init: Compile(optimized = not debugging and new scope) + S-->>D: delegates per clause, locals tuple + D->>D: Prepare (FunctionContext), DoBegin, ProcessRecord, Complete + D->>R: RunClause -> clause(functionContext) + R-->>D: output written to the output pipe + D-->>L: pipeline output to Out-Default and host +``` + +Ordered steps (`FACT-SRC` unless marked): + +1. **Process start**: `ManagedPSEntry.Main` → `UnmanagedPSEntry.Start`: early start-up warm-up, culture setup, `ConsoleHost.ParseCommandLine`, default initial session state, `ConsoleHost.Start` ([layered-architecture](../00-overview/layered-architecture.md) §4). +2. **Runspace loop**: `ConsoleHost.DoRunspaceLoop(initialCommand, …)` creates a runspace for each session iteration (`CreateRunspace`), runs the initial command through `InitializeRunspaceHelper` → `Executor.ExecuteCommand(command, out exception, options)`; at the end it reads `$global:?` to compute the exit code (success only if `$?` is true and the runspace initialized) (src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoRunspaceLoop@L1643-L1700). +3. **Pipeline creation**: `Executor.ExecuteCommand(string, …)` calls `CreatePipeline(command, addToHistory)` and `ExecuteCommandHelper`; when the `AddOutputter` option is set and the pipeline has a single command, the command's error results are merged into output and an `Out-Default` command is appended so results are rendered (src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor.ExecuteCommandHelper@L335-L357). +4. **Pipeline invoke**: `LocalPipeline.InvokeHelper` raises pipeline state events, transcribes script commands, and creates a `PipelineProcessor` with `CreatePipelineProcessor`, which asks every `Command` for a command processor (`command.CreateCommandProcessor(executionContext, addToHistory, origin)`) (src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.CreatePipelineProcessor@L817-L900). The `PipelineProcessor` then drives the command processors (`UNVERIFIED`: body not read, see §19). +5. **Script text becomes a script block**: for a script command, `Command.CreateCommandProcessor` first rejects scripts when the language mode is `NoLanguage` and the origin is the runspace (`ScriptsNotAllowed`), then calls `AutomationEngine.ParseScriptBlock(text, addToHistory)`: `EngineParser.Parse(fileName, script, null, out errors, ParseMode.Default)` (tokenize, parse, symbol resolution and semantic checks, [02.2-parser.md](02.2-parser.md), [02.4-semantic-analysis.md](02.4-semantic-analysis.md)); with `addToHistory` the parser records `$^`/`$$`; if any error exists it throws `IncompleteParseException` (when the first error is incomplete input, so a REPL can continue) or `ParseException(errors)`; otherwise `new ScriptBlock(ast, isFilter: false)` — **not** through the script block cache (src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127, src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540). +6. **Language-mode gate**: internal-origin commands run `FullLanguage`; `RestrictedLanguage` calls `CheckRestrictedLanguage` (the data-section style check over the whole script); `ConstrainedLanguage` is enforced at run time ([02.4-semantic-analysis.md](02.4-semantic-analysis.md) §14). +7. **Command processor**: a script block with `[CmdletBinding()]`/`[Parameter()]` becomes a `FunctionInfo` and a `CommandProcessor`; otherwise a `DlrScriptCommandProcessor` (named for the DLR-based compiled code). +8. **Compile**: the constructor `Init()` calls `ScriptBlock.Compile(optimized: _context._debuggingMode <= 0 && UseLocalScope)` (CF-CMP-003), which runs `CompiledScriptBlockData.Compile` → `Compiler.Compile` → per-clause `CompileTree` (interpreter delegate, CF-CMP-001) and creates the locals tuple (`MakeLocalsTuple`); with a local scope the tuple is installed on the new command scope ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md), [TIERING-POLICY.md](TIERING-POLICY.md)). +9. **Prepare / begin / process / complete**: `Prepare` stores `$MyInvocation`, `$PSScriptRoot`/`$PSCommandPath` and builds the `FunctionContext` (execution context, output pipe, locals tuple, script block, file, debugger flags, sequence points); `DoBegin` runs the `begin` clause once; `ProcessRecord` runs `process` (once per input object when input is expected, else once); `Complete` runs `end` (reading all input first when the block has no `process` clause) (src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.Prepare@L298-L313, src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.ProcessRecord@L352-L393). +10. **Run a clause**: `RunClause(clause, $_, input)` checks the stack depth, switches the language mode if the script block carries one, sets the scope origin, `EnterScope`, merges or redirects the error pipe, sets `$_` and `$input` (an enumerator over the input list), and calls `clause(_functionContext)`. Catches: `ExitException` (for script files sets `$LASTEXITCODE` and marks the pipeline failed when non-zero; otherwise rethrown), `FlowControlException` (rethrown — so `break`/`continue` escape a function), `RuntimeException` (`ManageScriptException`), other exceptions (wrapped). Finally restore error pipe, language mode and scope origin (src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L623). +11. **Execute**: the delegate is a `LightLambda`: interpreted by `Interpreter.Run` over the instruction array until the call counter triggers JIT (CF-CMP-001); statements run through the protocols of [IR-SPEC.md](IR-SPEC.md) §5 — binders choose operations ([BINDER-CATALOG.md](BINDER-CATALOG.md)), runtime helpers implement them ([RUNTIME-OPERATIONS.md](RUNTIME-OPERATIONS.md)); output is written to the output pipe (CF-PIP-001); errors go through `CheckActionPreference`. +12. **Render**: the appended `Out-Default` command receives the objects and the host writes them (`UNVERIFIED`: formatting and host UI not analyzed; Phase 3/4). +13. **Exit**: `DoRunspaceLoop` computes the exit code from `$?`; the runspace is closed. + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +`pwsh -c "1+1"` parses, creates one script command plus `Out-Default`, compiles the block, interprets it, writes `2` to the output pipe, and renders it. A parse error produces a `ParseException` whose first error id is the key reported in [ERROR-CATALOG.md](ERROR-CATALOG.md). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Parse errors stop before any command runs (the whole text is parsed first); runtime errors in the script are handled per statement (RUNTIME-OPERATIONS §6.1); an `ExitException` outside a script file is rethrown to the pipeline; `FlowControlException` crosses the command-processor boundary. + +## 10. Concurrency and threading + +Command line scripts run on the runspace pipeline thread; JIT promotion happens on the thread pool ([TIERING-POLICY.md](TIERING-POLICY.md) §7.5). + +## 11. Extension points and customization hooks + +Custom hosts (`ConsoleShell.Start` / `PSHost`), `InitialSessionState`, language mode, `Out-Default` replacement. + +## 12. Dependencies + +Host layer → engine layer ([layered-architecture](../00-overview/layered-architecture.md)). + +## 13. Performance notes + +Interactive and `-Command` text is parsed on every execution (no script cache on this path); compiled data live in the new `ScriptBlock` object and die with it. + +## 14. Security considerations (mandatory) + +Checks on this path, in order: `NoLanguage` rejection, parse, `CheckRestrictedLanguage` (restricted mode), compile-time security checks (`PerformSecurityChecks`: content scan, logging), run-time language-mode transitions in `RunClause`. A new shell should keep the same order: reject → parse → static policy → scan/log → run. + +## 15. Enterprise considerations + +Script block start/stop logging (`LogScriptBlockStart/End`), transcription of script commands in `LocalPipeline`, history ids on errors. + +## 16. Tests + +Host behavior tests: `test/powershell/Host/` (`data/tests.json`; mapping `UNVERIFIED`). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| command-line text is just a one-command pipeline plus `Out-Default` | `FACT-SRC` | one execution path for interactive and non-interactive use | +| script text parsed per execution, not cached on this path | `FACT-SRC` | simple; repeated identical commands re-parse | +| flow-control exceptions cross function boundaries | `FACT-SRC` | `break` in a function can end the caller's loop (SEMANTIC-QUIRKS) | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-LIFE-01** [REPLICATE] (area: NLD-RUN) Treat the command line as one script command followed by an output command in an ordinary pipeline, so interactive and file execution share a path. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor.ExecuteCommandHelper@L335-L357. +- **LSN-LIFE-02** [REPLICATE] (area: NLD-ERR) Parse the whole text before running anything and report incomplete input distinctly so a REPL can ask for another line. Evidence: src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127. +- **LSN-LIFE-03** [REPLICATE] (area: NLD-RUN) Run script blocks through a begin/process/end command processor that sets `$_`/`$input`, scopes, error pipe and language mode around each clause. Evidence: src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L623. +- **LSN-LIFE-04** [ADAPT] (area: NLD-RUN) Merge the `CommandProcessor`/`DlrScriptCommandProcessor` split into one script command processor. Why: the split exists for cmdlet-style binding of advanced functions; the observable begin/process/end behavior is the same. Evidence: src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540. + +## 19. Open questions / UNVERIFIED + +- `PipelineProcessor` execution, `Pipeline.Invoke` threading and `Out-Default` rendering were not read (Phase 3/4, T31/T36). +- Host: `-File`, `-EncodedCommand` and interactive loop differences. + +Verification: `ConsoleHost.DoRunspaceLoop` (L1643-L1700), `Executor.ExecuteCommandHelper` (L335-L357), `LocalPipeline.InvokeHelper` start and `CreatePipelineProcessor` (L279-L340, L817-L900), `Command.CreateCommandProcessor` (L450-L540), `AutomationEngine.ParseScriptBlock` (L91-L126), `DlrScriptCommandProcessor` (L232-L625) were read. diff --git a/docs-out/02-language/ORACLE-PROTOCOL.md b/docs-out/02-language/ORACLE-PROTOCOL.md new file mode 100644 index 0000000..5e9f44d --- /dev/null +++ b/docs-out/02-language/ORACLE-PROTOCOL.md @@ -0,0 +1,100 @@ +--- +id: LANG-ORACLE-001 +title: Oracle protocol +kind: guide +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [] +mirror_cards: [] +generated_from: [] +--- + +# Oracle protocol + +> STATIC-ONLY — the oracle was **waived** for this run (no .NET SDK 10.0.403 and no `pwsh` on the study host; backlog VB-001). This document records what the protocol would be, so a later run on a suitable host can execute it. Nothing here was executed. + +## 1. Purpose and scope [SPEC] + +How to obtain golden samples ([GOLDEN-SAMPLES.md](GOLDEN-SAMPLES.md)): run snippets through a real `pwsh` 7.6.x built from the pinned commit (or the matching release), capture tokens, AST, results and errors, normalize them, and diff against the claims of the study and later against MicroteamShell. + +## 2. Source map + +Planned tools: `docs-out/tools/oracle-harness` (not authored), `docs-out/tools/verify-with-pwsh.ps1` (not authored; the Lite prompt names it). + +## 3. Responsibilities and non-responsibilities + +Specifies commands and normalization; does not execute. + +## 4. Key types and contracts [IMPL] + +Inputs: a snippet list (from [REFERENCE-TEST-CORPUS.md](REFERENCE-TEST-CORPUS.md) and [SEMANTIC-QUIRKS.md](SEMANTIC-QUIRKS.md)); outputs: one JSON record per snippet ([GOLDEN-SAMPLES.md](GOLDEN-SAMPLES.md) §4). + +## 5. Data model + +Normalization: replace absolute paths and GUIDs, fix culture to invariant (and repeat selected snippets under a second culture), record `$PSVersionTable`, OS and time zone, serialize objects with type names and property values (depth-limited), sort hashtable keys. + +## 6. Control flow + +1. Build or install `pwsh` 7.6.x; verify `git rev-parse HEAD` equals the pin (the pin in `.work/pin.json`). +2. For each snippet: tokenize and parse with `[System.Management.Automation.Language.Parser]::ParseInput` (tokens, AST type names, errors); execute in a fresh runspace with `-NoProfile`; capture success output, error records, other streams, `$?`, `$LASTEXITCODE`. +3. Force execution modes: interpreted vs compiled by repeating a function call 40 times (JIT after the 33rd call) and a loop 20 times; optimized vs unoptimized by dot-sourcing; Constrained Language via `$ExecutionContext.SessionState.LanguageMode`. A mode that cannot be forced is reported as such (prompt 5.4). +4. Write the record; mark the matching claims verified. + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +Not applicable. + +## 9. Error handling and edge cases + +Snippets that hang or prompt are killed after a timeout and recorded as such. + +## 10. Concurrency and threading + +Each snippet runs in its own process for isolation. + +## 11. Extension points and customization hooks + +None. + +## 12. Dependencies + +[TIERING-POLICY.md](TIERING-POLICY.md) §7 (thresholds used to force modes), [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md) (what to compare across modes). + +## 13. Performance notes + +Not applicable. + +## 14. Security considerations (mandatory) + +Snippets run arbitrary code: use a disposable scratch copy, no network, no credentials, and never run repository scripts other than the snippets. + +## 15. Enterprise considerations + +Record tool versions and hashes with every run. + +## 16. Tests + +Not applicable. + +## 17. Design decisions and trade-offs + +Waived for this run by the owner; static reading plus the repository's own tests is the interim evidence. + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-ORACLE-01** [ADAPT] (area: NLD-TOOL) Use the recorded oracle outputs as the expected values of the new interpreter's conformance suite and re-run the oracle for each new quirk. Why: same coverage with an automated diff instead of manual review. Evidence: `test/powershell/Language/Operators/` (existing assertion style). + +## 19. Open questions / UNVERIFIED + +Everything in this file is unexecuted (VB-001). + +Verification: none (design only). diff --git a/docs-out/02-language/REFERENCE-TEST-CORPUS.md b/docs-out/02-language/REFERENCE-TEST-CORPUS.md new file mode 100644 index 0000000..9fe2ff8 --- /dev/null +++ b/docs-out/02-language/REFERENCE-TEST-CORPUS.md @@ -0,0 +1,242 @@ +--- +id: LANG-TESTCORPUS-001 +title: Reference test corpus +kind: catalog +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [test/powershell/Language] +mirror_cards: [] +generated_from: [data/tests.json] +--- + +# Reference test corpus + +> STATIC-ONLY — the repository's own tests are the interim behavior evidence (no golden samples). Tables are `GENERATED` by `docs-out/tools/gen_coord.py` from `data/tests.json` (`APPROX`: Pester `Describe`/`Context`/`It` counted by regex; xUnit `[Fact]`/`[Theory]` by regex). Test files were **inventoried, not read**: they are the model for the new product's specification-test suite and the source of the "covering tests" column of the feature matrix. + +## 1. Purpose and scope [SPEC] + +Existing parser and language tests under `test/` that specify language behavior, grouped by feature folder with paths and counts. The Language Pester folders contain 81 files and 1249 `It` blocks. + +## 2. Source map + +`test/powershell/Language/{Parser,Operators,Scripting,Classes,Interop}`, selected `test/powershell/engine/**`, `test/xUnit/csharp/test_Binders.cs`, `test_AstDefaultVisit.cs`. + +## 3. Responsibilities and non-responsibilities + +Inventory only; assertions are not reproduced (clean-room: describe, do not copy). + +## 4. Key types and contracts [IMPL] + +Pester 5 structure: `Describe` → `Context` → `It`; tags select CI priority. xUnit tests exercise internal APIs (binders, AST visitors). + +## 5. Data model — inventory + + +| Folder | Files | Describe | Context | It | +|---|---:|---:|---:|---:| +| `test/powershell/Language/Classes` | 10 | 42 | 26 | 158 | +| `test/powershell/Language/Interop` | 2 | 2 | 1 | 28 | +| `test/powershell/Language/Operators` | 7 | 10 | 16 | 130 | +| `test/powershell/Language/Parser` | 15 | 72 | 20 | 247 | +| `test/powershell/Language/Scripting` | 47 | 82 | 124 | 686 | +| **total Language** | **81** | **208** | **187** | **1249** | + +Files (path, Describe/Context/It counts, first Describe names; `APPROX` regex extraction): + +| File | D | C | It | First Describe names | +|---|---:|---:|---:|---| +| `test/powershell/Language/Classes/ProtectedAccess.Tests.ps1` | 3 | 0 | 35 | Protected Member Access - w/ default ctor; Protected Member Access - w/ non-default ctor | +| `test/powershell/Language/Classes/Scripting.Classes.Attributes.Tests.ps1` | 3 | 11 | 24 | Attributes Test; Type resolution with attributes | +| `test/powershell/Language/Classes/Scripting.Classes.BasicParsing.Tests.ps1` | 25 | 3 | 61 | Positive Parse Properties Tests; Negative Parsing Tests | +| `test/powershell/Language/Classes/Scripting.Classes.Break.Tests.ps1` | 1 | 4 | 7 | Break statements with classes | +| `test/powershell/Language/Classes/Scripting.Classes.Exceptions.Tests.ps1` | 3 | 3 | 14 | Exceptions flow for classes; Exception error position | +| `test/powershell/Language/Classes/Scripting.Classes.MiscOps.Tests.ps1` | 1 | 3 | 6 | Misc Test | +| `test/powershell/Language/Classes/Scripting.Classes.Modules.Tests.ps1` | 3 | 0 | 4 | PSModuleInfo.GetExportedTypeDefinitions(); use of a module from two runspaces | +| `test/powershell/Language/Classes/Scripting.Classes.NoRunspaceAffinity.Tests.ps1` | 1 | 0 | 1 | Class can be defined without Runspace affinity | +| `test/powershell/Language/Classes/Scripting.Classes.RunPath.Tests.ps1` | 1 | 0 | 1 | Script with a class definition run path | +| `test/powershell/Language/Classes/Scripting.Classes.StaticMethod.Tests.ps1` | 1 | 2 | 5 | Additional static method tests | +| `test/powershell/Language/Interop/DotNet/DotNetAPI.Tests.ps1` | 1 | 0 | 5 | DotNetAPI | +| `test/powershell/Language/Interop/DotNet/DotNetInterop.Tests.ps1` | 1 | 1 | 23 | Handle ByRef-like types gracefully | +| `test/powershell/Language/Operators/ComparisonOperator.Tests.ps1` | 2 | 0 | 13 | ComparisonOperator; Bytewise Operator | +| `test/powershell/Language/Operators/NullConditional.Tests.ps1` | 2 | 6 | 52 | NullCoalesceOperations; NullConditionalMemberAccess | +| `test/powershell/Language/Operators/PipelineChainOperator.Tests.ps1` | 1 | 2 | 7 | Experimental Feature: && and \|\| operators - Feature-Enable | +| `test/powershell/Language/Operators/RangeOperator.Tests.ps1` | 1 | 3 | 17 | Range Operator | +| `test/powershell/Language/Operators/ReplaceOperator.Tests.ps1` | 2 | 2 | 12 | Replace Operator; Culture-invariance tests for -split and -replace | +| `test/powershell/Language/Operators/SplitOperator.Tests.ps1` | 1 | 3 | 22 | Split Operator | +| `test/powershell/Language/Operators/TernaryOperator.Tests.ps1` | 1 | 0 | 7 | Using of ternary operator | +| `test/powershell/Language/Parser/Ast.Tests.ps1` | 1 | 0 | 4 | The SafeGetValue method on AST returns safe values | +| `test/powershell/Language/Parser/AutomaticVariables.Tests.ps1` | 1 | 0 | 2 | Automatic variable $input | +| `test/powershell/Language/Parser/BNotOperator.Tests.ps1` | 2 | 0 | 12 | bnot on enums; bnot on integral types | +| `test/powershell/Language/Parser/Conversions.Tests.ps1` | 5 | 1 | 24 | conversion syntax; Type resolution should prefer assemblies in powershell assem | +| `test/powershell/Language/Parser/ExtensibleCompletion.Tests.ps1` | 10 | 0 | 3 | Factory based extensible completion; Script block based extensible completion | +| `test/powershell/Language/Parser/LanguageAndParser.TestFollowup.Tests.ps1` | 12 | 0 | 23 | Clone array; Set fields through PSMemberInfo | +| `test/powershell/Language/Parser/LineContinuance.Tests.ps1` | 1 | 6 | 28 | Line Continuance | +| `test/powershell/Language/Parser/MethodInvocation.Tests.ps1` | 2 | 0 | 12 | Generic Method invocation; Interface inheritance with remoting proxies | +| `test/powershell/Language/Parser/ParameterBinding.Tests.ps1` | 3 | 0 | 14 | Argument transformation attribute on optional argument with ; Custom type conversion in parameter binding | +| `test/powershell/Language/Parser/Parser.Tests.ps1` | 1 | 9 | 85 | ParserTests (admin\monad\tests\monad\src\engine\core\ParserT | +| `test/powershell/Language/Parser/Parsing.Tests.ps1` | 27 | 0 | 14 | for statement parsing; foreach statement parsing | +| `test/powershell/Language/Parser/RedirectionOperator.Tests.ps1` | 4 | 1 | 7 | Redirection operator now supports encoding changes; File redirection mixed with Out-Null | +| `test/powershell/Language/Parser/TypeAccelerator.Tests.ps1` | 1 | 2 | 5 | Type accelerators | +| `test/powershell/Language/Parser/UsingAssembly.Tests.ps1` | 1 | 1 | 8 | Using assembly | +| `test/powershell/Language/Parser/UsingNamespace.Tests.ps1` | 1 | 0 | 6 | Using Namespace | +| `test/powershell/Language/Scripting/ActionPreference.Tests.ps1` | 2 | 7 | 36 | Tests for (error, warning, etc) action preference; ActionPreference.Break tests | +| `test/powershell/Language/Scripting/Adapter.Tests.ps1` | 2 | 0 | 5 | Interface static members; ByRef property | +| `test/powershell/Language/Scripting/Array.Tests.ps1` | 2 | 0 | 15 | ArrayExpression Tests; ArrayLiteral Tests | +| `test/powershell/Language/Scripting/CheckRestrictedlanguage.Tests.ps1` | 1 | 0 | 12 | Test restricted language check method on scriptblocks | +| `test/powershell/Language/Scripting/CleanBlockErrorHandling.Tests.ps1` | 3 | 0 | 49 | Error handling within a single 'Clean' block; Multiple errors from 'Clean' and another named block | +| `test/powershell/Language/Scripting/CmdletDeclaration.Tests.ps1` | 1 | 0 | 1 | Cmdlet declaration statement | +| `test/powershell/Language/Scripting/CommandErrorHandling.Tests.ps1` | 1 | 3 | 22 | Command error handling in general | +| `test/powershell/Language/Scripting/CommonParameters.Tests.ps1` | 1 | 10 | 22 | Common parameters support for script cmdlets | +| `test/powershell/Language/Scripting/ConstrainedLanguageMode.Tests.ps1` | 1 | 0 | 1 | Test constrained language mode | +| `test/powershell/Language/Scripting/Debugging/DebuggerCommand.Tests.ps1` | 3 | 9 | 51 | Basic debugger command tests; Simple debugger stepping command tests | +| `test/powershell/Language/Scripting/Debugging/DebuggerScriptTests.Tests.ps1` | 9 | 1 | 22 | Breakpoints set on custom FileSystem provider files should w; Tests line breakpoints on dot-sourced files | +| `test/powershell/Language/Scripting/Debugging/Debugging.Tests.ps1` | 3 | 9 | 19 | Basic debugger tests; Breakpoints when set should be hit | +| `test/powershell/Language/Scripting/Debugging/DebuggingInHost.Tests.ps1` | 1 | 0 | 4 | Tests Debugger GetCallStack() on runspaces when attached to | +| `test/powershell/Language/Scripting/Delegates.Tests.ps1` | 1 | 0 | 8 | Test for conversion b/w script block and delegate | +| `test/powershell/Language/Scripting/DeserializedMethods.Tests.ps1` | 1 | 0 | 1 | DeserializedMethods | +| `test/powershell/Language/Scripting/DeserializedTypeConversion.Tests.ps1` | 1 | 6 | 12 | Tests conversion of deserialized types to original type usin | +| `test/powershell/Language/Scripting/DollarHook.Tests.ps1` | 1 | 1 | 4 | Tests for setting $? for execution success | +| `test/powershell/Language/Scripting/Dynamicparameters.Tests.ps1` | 1 | 0 | 5 | Dynamic parameter support in script cmdlets. | +| `test/powershell/Language/Scripting/ErrorHistoryId.Tests.ps1` | 1 | 0 | 3 | Error HistoryId Tests | +| `test/powershell/Language/Scripting/ErrorPosition.Tests.ps1` | 1 | 0 | 2 | Error position Tests | +| `test/powershell/Language/Scripting/ForeachParallel.Tests.ps1` | 1 | 5 | 10 | Parallel foreach syntax | +| `test/powershell/Language/Scripting/Generics.Tests.ps1` | 1 | 0 | 10 | Generics support | +| `test/powershell/Language/Scripting/HashtableToPSCustomObjectConversion.Tests.ps1` | 2 | 0 | 11 | Tests for hashtable to PSCustomObject conversion; Error message with settable Property information | +| `test/powershell/Language/Scripting/I18n/I18n.Tests.ps1` | 1 | 6 | 16 | Testing of script internationalization | +| `test/powershell/Language/Scripting/Indexer.Tests.ps1` | 1 | 0 | 7 | Tests for indexers | +| `test/powershell/Language/Scripting/LineEndings.Tests.ps1` | 1 | 0 | 2 | Line endings | +| `test/powershell/Language/Scripting/MyInvocation.Tests.ps1` | 1 | 2 | 8 | Testing of MyInvocation | +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandArguments.Tests.ps1` | 5 | 0 | 15 | Behavior is specific for each platform; tests for multiple languages and extensions | +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandPathUpdate.Tests.ps1` | 1 | 2 | 9 | Path update for package managers | +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandProcessor.Tests.ps1` | 6 | 0 | 15 | Native pipeline should have proper encoding; native commands with pipeline | +| `test/powershell/Language/Scripting/NativeExecution/NativeLinuxCommands.Tests.ps1` | 2 | 0 | 8 | NativeLinuxCommands; Scripts with extensions | +| `test/powershell/Language/Scripting/NativeExecution/NativeStreams.Tests.ps1` | 2 | 1 | 6 | Native streams behavior with PowerShell; piping powershell objects to finished native executable | +| `test/powershell/Language/Scripting/NativeExecution/NativeUnixGlobbing.Tests.ps1` | 1 | 0 | 17 | Native UNIX globbing tests | +| `test/powershell/Language/Scripting/NativeExecution/NativeWindowsTildeExpansion.Tests.ps1` | 1 | 0 | 3 | Native Windows tilde expansion tests | +| `test/powershell/Language/Scripting/OrderedAttributeForHashTables.Tests.ps1` | 1 | 3 | 4 | Test for cmdlet to support Ordered Attribute on hash literal | +| `test/powershell/Language/Scripting/OutErrorVariable.Tests.ps1` | 3 | 1 | 15 | Tests OutVariable only; Test ErrorVariable only | +| `test/powershell/Language/Scripting/PSSerializer.Tests.ps1` | 1 | 0 | 3 | Tests for lossless rehydration of serialized types. | +| `test/powershell/Language/Scripting/ParameterBinding.Tests.ps1` | 1 | 2 | 40 | Tests for parameter binding | +| `test/powershell/Language/Scripting/PipelineBehaviour.Tests.ps1` | 2 | 5 | 29 | Function Pipeline Behaviour; Other Pipeline Behaviour | +| `test/powershell/Language/Scripting/PipelineStoppedToken.Tests.ps1` | 1 | 0 | 2 | PipelineStopToken tests | +| `test/powershell/Language/Scripting/Requires.Tests.ps1` | 2 | 7 | 8 | Requires tests; #requires -Modules | +| `test/powershell/Language/Scripting/ScriptHelp.Tests.ps1` | 3 | 28 | 92 | get-help HelpFunc1; get-help file | +| `test/powershell/Language/Scripting/Scripting.Followup.Tests.ps1` | 1 | 0 | 10 | Scripting.Followup.Tests | +| `test/powershell/Language/Scripting/SuppressAnsiEscapeSequence.Tests.ps1` | 1 | 2 | 6 | $env:__SuppressAnsiEscapeSequences tests | +| `test/powershell/Language/Scripting/SwitchParallel.Tests.ps1` | 1 | 3 | 5 | Parallel switch syntax | +| `test/powershell/Language/Scripting/Trap.Tests.ps1` | 1 | 1 | 4 | Test trap | +| `test/powershell/Language/Scripting/TryCatch.Tests.ps1` | 1 | 10 | 37 | Test try/catch | + +Engine tests that also specify language behavior (selected by path keywords, `APPROX`): + +| File | It | +|---|---:| +| `test/powershell/engine/Api/BasicEngine.Tests.ps1` | 9 | +| `test/powershell/engine/Api/GetMethodInfoOverloadDefinition.Tests.ps1` | 11 | +| `test/powershell/engine/Api/GetNewClosure.Tests.ps1` | 2 | +| `test/powershell/engine/Api/InitialSessionState.Tests.ps1` | 8 | +| `test/powershell/engine/Api/LanguagePrimitive.Tests.ps1` | 23 | +| `test/powershell/engine/Api/PSCommand.Tests.ps1` | 4 | +| `test/powershell/engine/Api/ProxyCommand.Tests.ps1` | 3 | +| `test/powershell/engine/Api/Serialization.Tests.ps1` | 7 | +| `test/powershell/engine/Api/SuspiciousContentChecker.Tests.ps1` | 1 | +| `test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1` | 9 | +| `test/powershell/engine/Api/TypeInference.Tests.ps1` | 202 | +| `test/powershell/engine/Basic/Assembly.LoadBytesAndLoadFile.Tests.ps1` | 2 | +| `test/powershell/engine/Basic/Assembly.LoadFrom.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/Assembly.LoadNative.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/Assembly.LoadWithPartialName.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/Assembly.LoadedInSeparateALC.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/Attributes.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/CLRBinding.Tests.ps1` | 12 | +| `test/powershell/engine/Basic/CommandDiscovery.Tests.ps1` | 15 | +| `test/powershell/engine/Basic/Credential.Tests.ps1` | 2 | +| `test/powershell/engine/Basic/DefaultCommands.Tests.ps1` | 6 | +| `test/powershell/engine/Basic/Encoding.Tests.ps1` | 7 | +| `test/powershell/engine/Basic/GroupPolicySettings.Tests.ps1` | 6 | +| `test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1` | 8 | +| `test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1` | 10 | +| `test/powershell/engine/Basic/PropertyAccessor.Tests.ps1` | 4 | +| `test/powershell/engine/Basic/SemanticVersion.Tests.ps1` | 19 | +| `test/powershell/engine/Basic/StandardLibraryTypes.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/Telemetry.Tests.ps1` | 11 | +| `test/powershell/engine/Basic/TypeResolution.Tests.ps1` | 1 | +| `test/powershell/engine/Basic/ValidateAttributes.Tests.ps1` | 17 | +| `test/powershell/engine/ETS/TypeTable.Tests.ps1` | 4 | +| `test/powershell/engine/Remoting/RunspacePool.Tests.ps1` | 1 | + +xUnit tests on the language layer: + +| File | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_Binders.cs` | 1 | 0 | PSEnumerableBinderTests | +| `test/xUnit/csharp/test_AstDefaultVisit.cs` | 2 | 0 | MyICustomAstVisitor2, MyDefaultCustomAstVisitor2, MyAstVisitor2, AstDefaultVisitTests | +| `test/xUnit/csharp/test_CommandLineParser.cs` | 55 | 52 | PSCommandLineParserTests, TestDataSettingsFile, TestDataLastFile | + + +Feature mapping (by folder, `INFERENCE` from names): `Parser/` — tokenizer, parser, AST, semantic errors ([02.1](02.1-tokenizer.md)-[02.4](02.4-semantic-analysis.md)); `Operators/` — operator semantics ([OPERATOR-TABLE.md](OPERATOR-TABLE.md), [CONVERSION-RULES.md](CONVERSION-RULES.md)); `Scripting/` — flow control, functions, scoping, errors, pipelines, redirection ([02.5](02.5-compiler-and-engine.md), [SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md)); `Classes/` — class and enum features (EXTENDED); `Interop/` — .NET interop and ETS-related behavior. + +## 6. Control flow + +Not applicable. + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +Each `It` is a behavior statement; the new product should port the *intent* of the Language tests as its own specification tests, after the oracle confirms current behavior. + +## 9. Error handling and edge cases + +`APPROX` counts miss parameterized and generated `It` blocks (`parameterized_blocks` field in the dataset). + +## 10. Concurrency and threading + +Not applicable. + +## 11. Extension points and customization hooks + +Tags (`CI`, `Feature`, `Slow`) in the dataset tree. + +## 12. Dependencies + +[GOLDEN-SAMPLES.md](GOLDEN-SAMPLES.md), [ORACLE-PROTOCOL.md](ORACLE-PROTOCOL.md). + +## 13. Performance notes + +Not applicable. + +## 14. Security considerations (mandatory) + +Tests run scripts; they were not executed here. + +## 15. Enterprise considerations + +Not applicable. + +## 16. Tests + +This document is the test inventory. + +## 17. Design decisions and trade-offs + +Inventory by path and counts rather than by assertion keeps the study clean-room and cheap; the cost is that tests are not yet linked to individual claims (VB-005). + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-CORPUS-01** [REPLICATE] (area: NLD-TOOL) Organize the new shell's specification tests by the same feature folders (parser, operators, scripting, classes) and keep one behavior per test. Evidence: `test/powershell/Language/Parser/` (folder layout in `data/tests.json`). +- **LSN-CORPUS-02** [ADAPT] (area: NLD-TOOL) Write the new suite in the shell's own language plus a small C# harness, so tests run without PowerShell. Why: same coverage, no dependency on the reference implementation. Evidence: `test/xUnit/csharp/test_Binders.cs` (xUnit tests of internal binders). + +## 19. Open questions / UNVERIFIED + +- Per-claim linking of tests (VB-005). +- Tests for Phase 3 areas (modules, providers) are outside this table. + +Verification: counts come from `data/tests.json` through the generator; the number of Language files and `It` blocks is registered (`LANGUAGE_TEST_FILES`, `LANGUAGE_TEST_ITS`). diff --git a/docs-out/02-language/SCRIPT-BLOCKS.md b/docs-out/02-language/SCRIPT-BLOCKS.md index 24f7d0e..ca4ec73 100644 --- a/docs-out/02-language/SCRIPT-BLOCKS.md +++ b/docs-out/02-language/SCRIPT-BLOCKS.md @@ -42,7 +42,7 @@ Value semantics of script blocks: creation, identity, sharing, binding, invocati - `ScriptBlock.Create(string)` parses with a new `Parser` and caches by (file name, text); the internal overload taking an `ExecutionContext` additionally binds the block to the current module session state when the engine is inside a module (src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.Create@L90-L99). - A `ScriptBlock` object is a thin handle: `_scriptBlockData` (the shared `CompiledScriptBlockData`), a bound session state, and the language mode captured from the creating context. `Clone()` makes a new handle over the **same** data, so compiled delegates are shared between clones and between cached copies (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.TryGetCachedScriptBlock@L583-L601). - Properties: `Attributes`, `File`, `IsFilter`, `IsConfiguration`, `Module`, `StartPosition`, `LanguageMode`, `OutputType`, `SessionState` (src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock@L83-L1085). -- Invocation: `Invoke(args)` (returns a `Collection`), `InvokeReturnAsIs(args)` (result object as is), `InvokeWithContext(functions, variables, args)` (defines extra functions/variables in a new scope), `InvokeUsingCmdlet`, all funneling into `DoInvoke`/`DoInvokeReturnAsIs` → `InvokeWithPipeImpl` ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.1). `InvokeAsMemberFunction*` serve class methods. `GetDelegate`/`CreateDelegate` convert to a delegate type. +- Invocation: `Invoke(args)` (returns a `Collection`), `InvokeReturnAsIs(args)` (result object as is), `InvokeWithContext(functions, variables, args)` (defines extra functions/variables in a new scope), `InvokeUsingCmdlet`, all funneling into `DoInvoke`/`DoInvokeReturnAsIs` → `InvokeWithPipeImpl` ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.1); running a block *as a command* (`&`, `.`, function or script call, command line text) goes through a command processor instead ([LIFECYCLE-script-execution.md](LIFECYCLE-script-execution.md) §6 steps 7-10). `InvokeAsMemberFunction*` serve class methods. `GetDelegate`/`CreateDelegate` convert to a delegate type. ## 5. Data model diff --git a/docs-out/02-language/SEMANTIC-QUIRKS.md b/docs-out/02-language/SEMANTIC-QUIRKS.md new file mode 100644 index 0000000..6eb2e06 --- /dev/null +++ b/docs-out/02-language/SEMANTIC-QUIRKS.md @@ -0,0 +1,132 @@ +--- +id: LANG-QUIRKS-001 +title: Semantic quirks +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/runtime/Binding/Binders.cs, src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs, src/System.Management.Automation/engine/LanguagePrimitives.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [] +--- + +# Semantic quirks + +> STATIC-ONLY — behavior not run-verified; there are no golden samples, so every quirk is read from source (`FACT-SRC`) and its confirmation with `pwsh` is a backlog item. Each quirk has a verdict from the shared vocabulary: `REPLICATE` (part of the baseline), `ADAPT` (same behavior, different mechanism), `REDESIGN` / `AVOID` (a deviation *proposal*, exactly one `DEV-*` row, baseline stays until the owner approves). Counts: 32 quirks — 26 REPLICATE, 3 ADAPT, 3 AVOID, 0 REDESIGN (counted from the table by `tools/gen_coord.py`). The design canvas ([new-language-design-canvas.md](../09-synthesis/new-language-design-canvas.md)) consumes these verdicts. + +## 1. Purpose and scope [SPEC] + +Surprising behaviors that an implementer would get wrong from intuition: lexical ambiguities, parsing choices, type coercions, collection handling, output rules, scoping and error semantics. + +## 2. Source map + +Evidence is by link to the owning document section, which cites the source lines. + +## 3. Responsibilities and non-responsibilities + +Catalog only; rules are specified in the linked sections and in [LANGUAGE-SPEC-DRAFT.md](LANGUAGE-SPEC-DRAFT.md). + +## 4. Key types and contracts [IMPL] + +Not applicable. + +## 5. Data model — the quirks + +| ID | Quirk | Evidence | Golden sample | Verdict | DEV | Rationale | +|---|---|---|---|---|---|---| +| QRK-01 | The same text tokenizes differently by mode: `-eq` is a parameter in a command and an operator in an expression; keywords are recognized only in command mode | [02.1 §6.3](02.1-tokenizer.md) | none (STATIC-ONLY) | REPLICATE | | defines the language's command-versus-expression behavior | +| QRK-02 | `$a-b` is `$a` minus `b` in an expression but one word in a command; `-` is not a variable-name character | [02.1 §7.3](02.1-tokenizer.md) | none | REPLICATE | | needed for command arguments like `echo $a-b` | +| QRK-03 | `#requires` is recognized only when the letters follow `#` immediately, without checking the next character; comments similar to a signature header (edit distance < 10) forbid code after them | [02.1 §7.4](02.1-tokenizer.md) | none | AVOID | DEV-LEX-001 | tokenizer should stay a pure function; pragmas and signature trailers move out of the tokenizer | +| QRK-04 | `$(` inside a string is pre-scanned by counting parentheses only, so a `)` in a nested quoted string may end the scan early (`UNVERIFIED`) | [02.1 §7.2](02.1-tokenizer.md) | none | ADAPT | | scan quotes and comments too; valid scripts behave the same | +| QRK-05 | Hashtable values and assignment right sides are statements: `@{a = if ($x) {1} else {2}}`, `$x = if …`, `$x = 1 \| f` | [GRAMMAR §7, §3a](GRAMMAR.ebnf.md) | none | REPLICATE | | grammar-level baseline | +| QRK-06 | No terminator is needed after a block statement: `if ($x) {1} 2` is two statements; `1 2` is an error | [02.2 §6.3](02.2-parser.md) | none | REPLICATE | | CF-GRM-001 | +| QRK-07 | All binary operators are left-associative, comparison operators share one level with `-split`, `-join`, `-shl`; comma binds tighter than every binary operator (`1,2 -eq 3` compares an array) | [OPERATOR-TABLE §3](OPERATOR-TABLE.md) | none | REPLICATE | | CF-OP-001 | +| QRK-08 | A leading sign belongs to a number only at a unary position (`AllowSignedNumbers`); `1 -1` is a subtraction | [02.1 §6.4](02.1-tokenizer.md) | none | REPLICATE | | needed for correct arithmetic parsing | +| QRK-09 | Typing is left-operand-driven: `1 + "2"` is 3, `"1" + 2` is `"12"`; `"a" * 3` repeats; `array + x` appends | [OPERATOR-TABLE §4](OPERATOR-TABLE.md) | none | REPLICATE | | defines operator behavior | +| QRK-10 | `int` overflow becomes `double`; `int / int` is an `int` only when exact, else `double`; division by zero is a runtime error | [02.6 §7.2](02.6-types-and-operators.md) | none | REPLICATE | | numeric baseline | +| QRK-11 | A collection on the left of `-eq`, `-like`, `-match` etc. filters and returns elements instead of a bool; `$null -eq $x` is true only for `$null` | [02.6 §7.3](02.6-types-and-operators.md) | none | REPLICATE | | idiomatic filtering | +| QRK-12 | Truthiness: non-empty string is true (even `"false"`); `@(0)` is false but `@(1,2)` is true; a single-element list is judged by its element | [CONVERSION-RULES §7.3](CONVERSION-RULES.md) | none | REPLICATE | | `IsTrue` | +| QRK-13 | Output unrolls exactly one level; strings, hashtables and `$null` are never enumerated; `AutomationNull` writes nothing | [02.6 §7.1](02.6-types-and-operators.md) | none | REPLICATE | | CF-PIP-001 | +| QRK-14 | `return $x` does not return a value: it writes `$x` to the output pipe, and a function's result is everything written to its pipe | [02.5 §6.5](02.5-compiler-and-engine.md) | none | REPLICATE | | pipeline semantics | +| QRK-15 | A bare expression statement is output; assignments and `++`/`--` are not; `[void]` and `$null =` discard | [IR-SPEC §5](IR-SPEC.md) | none | REPLICATE | | statement output rule | +| QRK-16 | Errors in a statement are reported and the script continues with the next statement (`ErrorActionPreference` default `Continue`); `throw` and `Stop` terminate | [RUNTIME-OPERATIONS §6.1](RUNTIME-OPERATIONS.md) | none | REPLICATE | | shell-style resilience | +| QRK-17 | Reads search parent scopes (dynamic scoping) but writes always go to the current scope; `$x = 1` in a function never changes the caller's `$x` | [SCOPE §7.1](SCOPE-AND-VARIABLES.md) | none | REPLICATE | | CF-SCP-001 | +| QRK-18 | `switch` tests every clause for every item (no implicit break); `default` runs only if none matched | [02.5 §6.3](02.5-compiler-and-engine.md) | none | REPLICATE | | switch semantics | +| QRK-19 | `foreach` over a single non-enumerable value runs once; over `$null` (no enumerator) it does not run | [02.5 §6.3](02.5-compiler-and-engine.md) | none | REPLICATE | | loop semantics (`UNVERIFIED` for `$null`) | +| QRK-20 | Numeric literals used as command arguments keep their source text (`echo 0x10` passes `0x10`); numbers converted to strings use the literal text when they came from a literal | [COMPILATION-MAP](COMPILATION-MAP.md) `VisitCommand`, [CONVERSION-RULES §7.2](CONVERSION-RULES.md) | none | REPLICATE | | argument fidelity | +| QRK-21 | `hashtable + hashtable` builds a new table and throws on a shared key; the literal uses an ordinal comparer but `+` a culture comparer | [RUNTIME-OPERATIONS §6.3](RUNTIME-OPERATIONS.md), CON-011 | none | ADAPT | | keep merge/duplicate behavior, use one comparer | +| QRK-22 | `string * n` and array repetition are capped at 1,024 only in restricted language; a negative count throws `ArgumentOutOfRangeException` | [RUNTIME-OPERATIONS §6.2-6.3](RUNTIME-OPERATIONS.md) | none | ADAPT | | cap in every mode, raise a runtime error | +| QRK-23 | Postfix `++` on `$null` yields `0` as the expression value | [02.5](02.5-compiler-and-engine.md) notes (`CompileIncrementOrDecrement`) | none | REPLICATE | | backward compatibility | +| QRK-24 | `?` is a plain word in command mode (alias `?`), an operator only in expression mode; `??` and `?:` exist only in expression mode | [02.1 §6.1](02.1-tokenizer.md) | none | REPLICATE | | mode rule | +| QRK-25 | Keywords used as command arguments are plain words (`echo if`); in expression mode keywords are identifiers | [02.2 §7.4](02.2-parser.md), [02.1 §6.3](02.1-tokenizer.md) | none | REPLICATE | | mode rule | +| QRK-26 | In expression mode `2>x` tokenizes as the number `2` followed by `>`; stream numbers 1-6 only | [02.1 §7.5](02.1-tokenizer.md) | none | REPLICATE | | redirection tokenizing | +| QRK-27 | `break`/`continue` are exceptions across function boundaries: a `break` inside a function called from a loop ends the caller's loop | [LIFECYCLE §6 step 10](LIFECYCLE-script-execution.md), [02.5 §6.3](02.5-compiler-and-engine.md) | none | REPLICATE | | flow-control baseline (`UNVERIFIED`: caller-side catch not traced) | +| QRK-28 | Parsing has side effects: `using assembly` probes the file system, `configuration` can create a runspace, class and enum errors surface at compile time | [02.2 §14](02.2-parser.md), [02.4 §6](02.4-semantic-analysis.md) | none | AVOID | DEV-GRM-010 | parse should be pure | +| QRK-29 | `using module` is resolved while parsing by running `Get-Module -ListAvailable` | [02.4 §6](02.4-semantic-analysis.md) | none | AVOID | DEV-SEM-030 | module import belongs to a later phase | +| QRK-30 | `GetNewClosure` snapshots only variables of the current scope with no special options, by value | [SCRIPT-BLOCKS §6](SCRIPT-BLOCKS.md) | none | REPLICATE | | closure baseline | +| QRK-31 | `AllScope` variables are the same object in every scope; `$null`, `$true`, `$false` are such variables | [SCOPE §5](SCOPE-AND-VARIABLES.md) | none | REPLICATE | | scope baseline | +| QRK-32 | String-to-number conversion accepts the literal syntax (hex, binary, suffixes, `kb`); empty string converts to 0 | [CONVERSION-RULES §7.2](CONVERSION-RULES.md) | none | REPLICATE | | `Parser.ScanNumber` | + +## 6. Control flow + +Not applicable. + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +All rows are observable; none is run-verified. + +## 9. Error handling and edge cases + +Rows marked `UNVERIFIED` in their rationale are in the backlog (VB-002 to VB-023). + +## 10. Concurrency and threading + +Not applicable. + +## 11. Extension points and customization hooks + +Not applicable. + +## 12. Dependencies + +Documents linked in the evidence column. + +## 13. Performance notes + +Not applicable. + +## 14. Security considerations (mandatory) + +QRK-03, QRK-28 and QRK-29 are the security-relevant quirks: fuzzy signature detection, parse-time file and module access, and parse-time runspace creation. QRK-22 matters for denial of service. + +## 15. Enterprise considerations + +Not applicable. + +## 16. Tests + +Confirming each quirk is the first use of the oracle protocol ([ORACLE-PROTOCOL.md](ORACLE-PROTOCOL.md)); candidate tests: [REFERENCE-TEST-CORPUS.md](REFERENCE-TEST-CORPUS.md). + +## 17. Design decisions and trade-offs + +Most quirks are consequences of two decisions: left-operand-driven typing and output-as-side-effect. Replicating them is cheap if the evaluator follows the protocols; changing them breaks scripts. + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-QRK-01** [REPLICATE] (area: NLD-OP) Treat QRK-01 to QRK-27 and QRK-30 to QRK-32 as part of the baseline language and specify them with tests before implementing. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`. +- **LSN-QRK-02** [ADAPT] (area: NLD-OP) Keep the observable behavior of QRK-04, QRK-21 and QRK-22 with safer mechanisms (full scan, single comparer, caps in all modes). Evidence: `src/System.Management.Automation/engine/runtime/Operations/StringOps.cs::StringOps@L15-L107`. +- **LSN-QRK-03** [AVOID] (area: NLD-GRM) Do not reproduce parse-time side effects (QRK-28, QRK-29) or tokenizer-level pragma handling (QRK-03). Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168`. DEV: DEV-GRM-010 + +## 19. Open questions / UNVERIFIED + +Every row needs the oracle; QRK-04, QRK-19, QRK-27 are explicitly partial. + +Verification: each row was derived from a document section that cites source reads; verdict counts are computed from this table. diff --git a/docs-out/02-language/STAGE-CONTRACTS.md b/docs-out/02-language/STAGE-CONTRACTS.md new file mode 100644 index 0000000..01bd0f4 --- /dev/null +++ b/docs-out/02-language/STAGE-CONTRACTS.md @@ -0,0 +1,181 @@ +--- +id: LANG-STAGES-001 +title: Stage contracts +kind: guide +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [] +--- + +# Stage contracts + +> STATIC-ONLY — behavior not run-verified. The data contract at each pipeline boundary: types exchanged, ownership and lifetime, invariants, error channel, threading, and the natural place for a policy or security hook. Each contract ends with a `NEW-SHELL CONTRACT` recommendation (`INFERENCE`; verdict vocabulary in [SEMANTIC-QUIRKS.md](SEMANTIC-QUIRKS.md)). Map: [00-pipeline-map.md](00-pipeline-map.md). + +## 1. Purpose and scope [SPEC] + +Six boundaries: (B1) text → tokens, (B2) tokens → AST, (B3) AST → checks and annotations, (B4) AST → compiled artifact, (B5) runtime → values, (B6) values → output objects. These contracts are the spine of a re-implementation. + +## 2. Source map + +| Boundary | Producer | Consumer | Key members | +|---|---|---|---| +| B1 | src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L595-L5082 (file `tokenizer.cs`) | src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8187 | `NextToken`, `Mode`, `Resync` | +| B2 | `Parser` | all later stages | `ParseInput`, `ParseFile`, `ParseTask` | +| B3 | `SymbolResolver`, `SemanticChecks` | compiler | `PerformPostParseChecks` | +| B4 | `Compiler` | `ScriptBlock` invocation | `Compile`, `CompileTree` | +| B5 | binders, runtime operations | compiled code | `ConvertTo`, binder rules | +| B6 | compiled code | command processors, host | `Pipe.Add`, `CheckActionPreference` | + +## 3. Responsibilities and non-responsibilities + +Contracts only; behavior of each stage is in its own document. + +## 4. Key types and contracts [IMPL] + +### B1 — text → tokens + +- **Types**: `Token` and subclasses (`NumberToken`, `ParameterToken`, `VariableToken`, `StringLiteralToken`, `StringExpandableToken`, `LabelToken`, redirection tokens); input is a `string`; the tokenizer owns a line-start table ([02.1-tokenizer.md](02.1-tokenizer.md) §4, §7.6). +- **Ownership and lifetime**: tokens are created on demand and referenced by the parser (one pushed-back token) and optionally by a caller-supplied token list; a rescan after a mode change discards the pending token ([02.1-tokenizer.md](02.1-tokenizer.md) §6.6). +- **Invariants**: extents are contiguous and refer to the original text (compensated for rewritten sub-expression text); `EndOfInput` is returned at the end; newlines are tokens; comments are not returned but can be saved. +- **Error channel**: not an exception; the tokenizer calls `Parser.ReportError`/`ReportIncompleteInput` and marks tokens `TokenInError` ([ERROR-CATALOG.md](ERROR-CATALOG.md)). +- **Threading**: one tokenizer per parser, single thread; the dynamic-keyword table is thread-local. +- **Hook point**: before tokenization (input size, encoding, source provenance). +- **NEW-SHELL CONTRACT**: `Tokenizer.Next(mode)` returns one immutable token with an extent; mode is an argument, not hidden state; the tokenizer reports diagnostics into a collection owned by the parse. Why: makes mode switching testable and removes the rescan hack's need for hidden state. + +### B2 — tokens → AST + +- **Types**: output `ScriptBlockAst` (root), `Token[]` (if requested), `ParseError[]`; entry `Parser.ParseInput(string, out Token[], out ParseError[])` ([02.2-parser.md](02.2-parser.md) §4). +- **Ownership**: the tree owns its nodes; each node has exactly one parent set at construction ([02.3-ast.md](02.3-ast.md) §4); the parser instance is discarded after the parse. +- **Invariants**: a script with errors still yields a tree (with `ErrorStatementAst`/`ErrorExpressionAst`); extents nest; hashtable values and assignment right sides are statements; terminator rules per CF-GRM-001. +- **Error channel**: `ParseError` (id = resource key, extent, message, `IncompleteInput`), de-duplicated and unlimited (CF-ERR-002); fatal parser exceptions become `ParseException(UnrecoverableParserError)`; `ScriptBlock.Create` throws `ParseException` for any error. +- **Threading**: a parser is single-threaded; on stack exhaustion the parse is retried once on a new thread. +- **Hook point**: right after parsing, before checks — policy on syntax (banned statements, size, nesting depth). +- **NEW-SHELL CONTRACT**: `Parse(text) → ParseResult { Ast, Tokens, Diagnostics }` that never throws for bad input and has a configurable nesting limit. + +### B3 — AST → checks and annotations + +- **Types**: in: the root `ScriptBlockAst`; out: additional `ParseError`s and annotations on nodes (`VariableExpressionAst.TupleIndex/Assigned`, `HasSuspiciousContent`, cached resolved types, `ScriptBlockAst.HadErrors/PostParseChecksPerformed`) ([02.4-semantic-analysis.md](02.4-semantic-analysis.md) §4-5). +- **Invariants**: runs only for `ParseMode.Default`; symbol resolution before semantic checks; class-definition errors that depend on Reflection.Emit appear later, at compile (CON-009). +- **Error channel**: the same `ParseError` list. +- **Threading**: visitors run on the parsing thread; module lookup for `using module` uses a thread-static `PowerShell`. +- **Hook point**: **the** place for static policy (restricted language checks for `data` sections, Constrained Language rules, forbidden commands/types). +- **NEW-SHELL CONTRACT**: a pure `Check(ast, options) → diagnostics + SemanticModel` where annotations live in a side model keyed by node (DEV-AST-020), with no I/O. + +### B4 — AST → compiled artifact + +- **Types**: in: `ScriptBlockAst` (via `CompiledScriptBlockData`); out: one `Action` per named block and variant, plus locals tuple type, name → slot map, sequence points ([COMPILATION-MAP.md](COMPILATION-MAP.md), [IR-SPEC.md](IR-SPEC.md) §4). +- **Ownership and lifetime**: compiled data are shared by all clones of a script block and cached (up to 1,024 parsed scripts); delegates are published under locks ([TIERING-POLICY.md](TIERING-POLICY.md)). +- **Invariants**: the IR implements the protocols of [IR-SPEC.md](IR-SPEC.md) §5 (output pipe, `$?`, error-action resume, positions, flow control); both execution tiers are equivalent ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md)). +- **Error channel**: `ParseException` from class definition or security checks; compile exceptions propagate to the caller. +- **Threading**: compile under `lock`; tier promotion on the thread pool. +- **Hook point**: `PerformSecurityChecks` before compiling (content scan, logging), and the per-block compile call. +- **NEW-SHELL CONTRACT**: `Compile(ast, variant) → Executable` where `Executable.Run(Frame)` is the only entry; the executable may be the AST itself (tree-walker) ([INTERPRETER-DESIGN-SPACE.md](INTERPRETER-DESIGN-SPACE.md)). + +### B5 — runtime → values + +- **Types**: arbitrary .NET objects, `PSObject` wrappers, `AutomationNull.Value`; operations go through binders and runtime helpers ([BINDER-CATALOG.md](BINDER-CATALOG.md), [RUNTIME-OPERATIONS.md](RUNTIME-OPERATIONS.md)); conversions by [CONVERSION-RULES.md](CONVERSION-RULES.md). +- **Invariants**: left-operand-driven typing; null/`AutomationNull` distinction; enumeration rule CF-PIP-001; variables via scope rules CF-SCP-001. +- **Error channel**: `RuntimeException` (wrapping `ErrorRecord` with script position), `PSInvalidCastException`, flow-control exceptions (`BreakException`, `ContinueException`, `ReturnException`, `ExitException`). +- **Threading**: one runspace thread executes a pipeline; caches are concurrent. +- **Hook point**: binder construction and conversion selection (language mode checks happen here); member-invocation logging. +- **NEW-SHELL CONTRACT**: a `Value` model with explicit `Null` and `NoValue`, one `Convert(value, type, context)` entry, and one `Invoke(target, member, args, context)` entry through which every host call passes (the policy choke point). + +### B6 — values → output objects + +- **Types**: the output pipe (`Pipe`) receives objects; non-enumerable values are written whole, enumerables unrolled one level (CF-PIP-001); errors become `ErrorRecord`s on the error stream via `CheckActionPreference` ([RUNTIME-OPERATIONS.md](RUNTIME-OPERATIONS.md) §6.1). +- **Invariants**: a statement's value is output only for bare expressions and commands; temporary pipes capture output for conditions, assignments, `$()` and `@()` ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.5). +- **Error channel**: `ErrorRecord` (category, target, script position, history id) written to the error stream or thrown. +- **Threading**: pipes belong to the executing pipeline; stopping is cooperative (`CheckForInterrupts`). +- **Hook point**: pipe writers and the host UI (formatting), error reporting. +- **NEW-SHELL CONTRACT**: an `OutputSink` interface with `Write(object)`, `WriteError(ErrorRecord)` and capture scopes; the formatting layer is outside the language core. + +## 5. Data model + +See the type lists per boundary; the shared objects are `Token`, `Ast` + `ParseError`, `FunctionContext` ([IR-SPEC.md](IR-SPEC.md) §4.2), `ExecutionContext` ([execution-context-and-session-state.md](../03-runtime/execution-context-and-session-state.md)) and `ErrorRecord`. + +## 6. Control flow + +```mermaid +sequenceDiagram + participant H as Host + participant P as Parser + participant K as Checks + participant C as Compiler + participant X as Execution + H->>P: B1/B2 text -> tokens -> AST + P-->>H: AST + ParseError[] + H->>K: B3 AST + K-->>H: errors + annotations + H->>C: B4 AST + C-->>H: executable blocks + H->>X: B5 run with ExecutionContext + X-->>H: B6 output objects and ErrorRecords +``` + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +Not applicable (interfaces). + +## 9. Error handling and edge cases + +Three error channels: `ParseError` before execution, `RuntimeException` during execution, `ErrorRecord` on the error stream; flow-control exceptions are control signals, not errors. + +## 10. Concurrency and threading + +One thread per pipeline; compile and tier promotion are concurrent-safe; parser objects are not shared. + +## 11. Extension points and customization hooks + +Hook points per boundary above; dynamic keywords at B2/B3; type accelerators and ETS at B5. + +## 12. Dependencies + +All 02-language documents. + +## 13. Performance notes + +Boundaries B1-B2 cost O(n); B4 cost is amortized by caching; B5 uses per-site caches. + +## 14. Security considerations (mandatory) + +The hook points are the answer to "where can policy be enforced": B3 (static), B4 (content scanning), B5 (dynamic: every member call and conversion), B6 (output filtering). PowerShell enforces language mode at B3 (`data` sections), B5 (binders, conversion gate) and B4 (`PerformSecurityChecks`). + +## 15. Enterprise considerations + +Diagnostics exist at every boundary (ETW parse/compile events, error ids, script block logging); a new shell should add a structured trace per boundary. + +## 16. Tests + +Boundary-level tests are the Language Pester tests ([REFERENCE-TEST-CORPUS.md](REFERENCE-TEST-CORPUS.md)). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| tokenizer driven by the parser through a mode | `FACT-SRC` | context-sensitive tokenizing; tokens not reusable in isolation | +| AST carries annotations for later stages | `FACT-SRC` | convenient, couples stages | +| runtime errors carry script positions through sequence points | `FACT-SRC` | good diagnostics; needs position bookkeeping at run time | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-STAGES-01** [REPLICATE] (area: NLD-TOOL) Keep the six boundaries as explicit interfaces with the three error channels (diagnostics, runtime exception, error record). Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`. +- **LSN-STAGES-02** [REPLICATE] (area: NLD-SEC) Place static policy at B3, content policy at B4, and a single invoke/convert choke point at B5. Evidence: `src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.PerformSecurityChecks@L197-L283`. +- **LSN-STAGES-03** [ADAPT] (area: NLD-GRM) Pass tokenizer mode as an argument and keep annotations in a side model. Why: same behavior, removes hidden state. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`. + +## 19. Open questions / UNVERIFIED + +- `Pipe` internals and command-processor boundaries (Phase 3). +- Threading rules of runspace pipelines are summarized from the context document only. + +Verification: every contract cites the owning document section, which cites source; hook points are `INFERENCE` unless a document section is cited. diff --git a/docs-out/02-language/TIERING-POLICY.md b/docs-out/02-language/TIERING-POLICY.md index 4f6f909..182245d 100644 --- a/docs-out/02-language/TIERING-POLICY.md +++ b/docs-out/02-language/TIERING-POLICY.md @@ -73,9 +73,9 @@ flowchart TD ### 7.1 Which variant (CF-CMP-003) -**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used for script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables. +**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used by the command processors that run script blocks (`DlrScriptCommandProcessor` constructor) and script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables. -Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.InvokeWithPipeImpl@L935-L959 (`runOptimized = context._debuggingMode <= 0 && createLocalScope`), src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552 (constructor), src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847, src/System.Management.Automation/engine/parser/VariableAnalysis.cs::FindAllVariablesVisitor@L44-L323. +Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.InvokeWithPipeImpl@L961-L1263 (`runOptimized = context._debuggingMode <= 0 && createLocalScope`), src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.Init@L274-L296 (`Compile(optimized: _context._debuggingMode <= 0 && UseLocalScope)`), src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552 (constructor), src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847, src/System.Management.Automation/engine/parser/VariableAnalysis.cs::FindAllVariablesVisitor@L44-L323. ### 7.2 Interpret versus JIT (CF-CMP-001) @@ -115,7 +115,7 @@ None public. `InternalTestHooks` can disable the script block cache. `CompileSyn ## 12. Dependencies -Inbound: `ScriptBlock` invocation, `PSScriptCmdlet`, `Compiler`. Outbound: `System.Linq.Expressions`, `ThreadPool`. +Inbound: `ScriptBlock` invocation, `DlrScriptCommandProcessor`, `PSScriptCmdlet`, `Compiler`. Outbound: `System.Linq.Expressions`, `ThreadPool`. ## 13. Performance notes diff --git a/docs-out/09-synthesis/deviation-register.md b/docs-out/09-synthesis/deviation-register.md index f2db493..f0de978 100644 --- a/docs-out/09-synthesis/deviation-register.md +++ b/docs-out/09-synthesis/deviation-register.md @@ -26,10 +26,11 @@ All rows are `PROPOSED` (the only status this run may write). `APPROVED`/`REJECT | DEV-CMD-121 | PROPOSED | WSMan client cmdlets and provider (assembly Microsoft.WSMan.Management) are part of the Windows command set. | Omit WSMan commands; remoting transport becomes pluggable behind one session abstraction. | No dependency on a Windows-only protocol stack in the base product. | WSMan-based automation breaks; an optional transport module could restore it. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-WSMAN-01 (10-command-modules/Microsoft.WSMan.Management.md) | | DEV-CMD-122 | PROPOSED | CIM/WMI cmdlets (assembly Microsoft.Management.Infrastructure.CimCmdlets) ship on Windows. | Provide CIM/WMI access only as an optional adapter module. | Keeps the base command set free of Windows management stacks. | Windows management scripts need the optional module. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CIM-01 (10-command-modules/Microsoft.Management.Infrastructure.CimCmdlets.md) | | DEV-CTX-081 | PROPOSED | ExecutionContext carries language mode and invalidates binder caches on change. | No language modes. | No cache invalidation logic. | No restricted modes. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CTX-04 (03-runtime/execution-context-and-session-state.md) | +| DEV-DESIGN-110 | PROPOSED | PowerShell compiles to expression trees and uses Reflection.Emit for classes. | MicroteamShell starts with a tree-walking evaluator and does not adopt Reflection.Emit or Expression.Compile as the core engine. | AOT-friendly, debuggable, small. | Lower peak throughput; classes (if added) are interpreter objects. | CF-CMP-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-DESIGN-03 (02-language/INTERPRETER-DESIGN-SPACE.md) | | DEV-ENG-040 | PROPOSED | Compilation of a top-level script block runs an AMSI scan, admin-block check and suspicious-content hashing. | Compilation is policy-free; an optional policy hook runs before execution. | Smaller engine core; platform-independent. | No AMSI integration or forced logging on suspicious content. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-ENG-07 (02-language/02.5-compiler-and-engine.md) | | DEV-EQV-061 | PROPOSED | Interpreter keeps typed arithmetic/comparison instruction families inherited from the DLR. | Do not port them; numeric semantics come from the evaluator's conversion rules. | Removes dead and defective code. | None observable (instructions unreachable from PowerShell IR). | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-EQV-03 (02-language/EQUIVALENCE-SPEC.md) | | DEV-FEAT-091 | PROPOSED | workflow/parallel/sequence/inlinescript parse and then fail in semantic checks; from/define/var are reserved. | Reject at parse time. | No dead syntax. | Different error text. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-FEAT-03 (02-language/02.7-language-features.md) | -| DEV-GRM-010 | PROPOSED | The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing. | Parsing is pure: directives are recorded in the AST and resolved in a later phase. | Deterministic, side-effect-free parse; safe for tooling and untrusted text. | Errors for a missing assembly appear at a later phase than in PowerShell. | CF-GRM-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PARSER-07 (02-language/02.2-parser.md) | +| DEV-GRM-010 | PROPOSED | The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing. | Parsing is pure: directives are recorded in the AST and resolved in a later phase. | Deterministic, side-effect-free parse; safe for tooling and untrusted text. | Errors for a missing assembly appear at a later phase than in PowerShell. | CF-GRM-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PARSER-07 (02-language/02.2-parser.md); LSN-QRK-03 (02-language/SEMANTIC-QUIRKS.md) | | DEV-LEX-001 | PROPOSED | Tokenizer detects signature-block lookalikes and processes #requires comment lines at parse time. | Omit both from the tokenizer; if wanted, handle pragmas in a dedicated post-tokenize pass. | Keeps the tokenizer a pure text-to-token function; removes security-sensitive fuzzy matching from the hot path. | Scripts using #requires get no version/module check; signed-script trailer detection unavailable. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOK-07 (02-language/02.1-tokenizer.md) | | DEV-RTOPS-051 | PROPOSED | Classes are emitted with Reflection.Emit and method bodies wrapped by ScriptBlockMemberMethodWrapper. | Classes are interpreter objects, not CLR types. | No Reflection.Emit dependency. | Classes are not usable from other .NET code. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTOPS-05 (02-language/RUNTIME-OPERATIONS.md) | | DEV-SB-090 | PROPOSED | ScriptBlock can become a PowerShell object and a steppable pipeline. | Omitted initially. | Less surface. | No GetPowerShell/GetSteppablePipeline. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SB-05 (02-language/SCRIPT-BLOCKS.md) | diff --git a/docs-out/09-synthesis/new-language-design-canvas.md b/docs-out/09-synthesis/new-language-design-canvas.md new file mode 100644 index 0000000..5172194 --- /dev/null +++ b/docs-out/09-synthesis/new-language-design-canvas.md @@ -0,0 +1,40 @@ +--- +id: SYN-CANVAS-001 +title: New-language design canvas (skeleton) +kind: catalog +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [] +mirror_cards: [] +generated_from: [data/lessons.json] +--- + +# New-language design canvas (skeleton) + +> STATIC-ONLY. Skeleton created in Phase 2 so that forward links resolve; Phase 7 fills the empty cells. Lesson counts per area and verdict are `GENERATED` by `docs-out/tools/gen_coord.py` from `data/lessons.json` (gate 6d compares them with the verdict counts of [SEMANTIC-QUIRKS](../02-language/SEMANTIC-QUIRKS.md) and the guidance sections). + +## 1. Canvas rows + +| Row | Area | REPLICATE | ADAPT | REDESIGN | AVOID | Baseline decision | Approved deviation | Status | +|---|---|---:|---:|---:|---:|---|---|---| +| NLD-LEX | Lexical structure and tokenizer modes | 4 | 2 | 0 | 1 | | | open | +| NLD-GRM | Grammar, AST and semantic checks | 10 | 6 | 0 | 4 | | | open | +| NLD-OP | Operators and precedence | 7 | 4 | 0 | 0 | | | open | +| NLD-TYP | Types, conversions and values | 6 | 2 | 0 | 0 | | | open | +| NLD-SCP | Scopes, variables and sessions | 11 | 3 | 0 | 0 | | | open | +| NLD-ERR | Errors, streams and recovery | 7 | 0 | 0 | 0 | | | open | +| NLD-PIP | Pipeline and output | 3 | 0 | 0 | 0 | | | open | +| NLD-MOD | Modules and packaging | 0 | 1 | 0 | 0 | | | open | +| NLD-CMD | Commands and parameter binding | 3 | 2 | 0 | 3 | | | open | +| NLD-SEC | Security, sandboxing and language modes | 6 | 5 | 0 | 5 | | | open | +| NLD-TOOL | Tooling, extension and testing | 12 | 9 | 0 | 1 | | | open | +| NLD-RUN | Execution strategy and runtime | 9 | 11 | 0 | 6 | | | open | +| **total** | | **78** | **45** | **0** | **20** | | | | + +Verification: counts rendered from `data/lessons.json` (extracted from the guidance sections by `tools/coordinator.py lessons`); the other columns are intentionally empty until Phase 7. + diff --git a/docs-out/CONTRADICTIONS.md b/docs-out/CONTRADICTIONS.md index a3443ad..90619ec 100644 --- a/docs-out/CONTRADICTIONS.md +++ b/docs-out/CONTRADICTIONS.md @@ -16,3 +16,4 @@ Rows are never resolved by silent rewording. Status: `OPEN | RESOLVED`. Final st | CON-010 | study prompt hint: `engine/lang/parserutils.cs` holds parser utilities | file holds runtime operator helpers (`ParserOps`), flow-control exceptions and `InterpreterError` (`src/System.Management.Automation/engine/lang/parserutils.cs@L23-L2010`) | prompt hint vs E1 | RESOLVED | source wins; documented as runtime helpers. | docs-out/02-language/02.5-compiler-and-engine.md | | CON-011 | `Compiler.BuildHashtable` creates hashtables with `OrdinalIgnoreCase` keys (`src/System.Management.Automation/engine/parser/Compiler.cs@L6533-L6546`) | `HashtableOps.Add` builds the result of `+` with `CurrentCultureIgnoreCase` (`src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs@L1431-L1456`) | E1 code vs E1 code | OPEN | observation recorded in RUNTIME-OPERATIONS §6.3; behavior differs only for culture-sensitive case folding (e.g. Turkish i); needs `pwsh` under such a culture. | docs-out/02-language/RUNTIME-OPERATIONS.md | | CON-012 | `StringOps.Multiply` comment: negative count "should be a runtime error" (`src/System.Management.Automation/engine/runtime/Operations/StringOps.cs@L27-L35`) | code throws `ArgumentOutOfRangeException` | E1 comment vs E1 code | RESOLVED | code is authoritative; documented in RUNTIME-OPERATIONS §6.3. | docs-out/02-language/RUNTIME-OPERATIONS.md | +| CON-013 | earlier study documents (02.5 §4, TIERING-POLICY §2, SCRIPT-BLOCKS §4) said `ScriptBlock.InvokeWithPipeImpl` is the one execution entry for functions, scripts, `&`, `.` and `Invoke` | reading `Command.CreateCommandProcessor` and `DlrScriptCommandProcessor` shows command invocation of script text, functions and scripts runs through command processors; `InvokeWithPipeImpl` serves the `ScriptBlock.Invoke*` API (`src/System.Management.Automation/engine/hostifaces/Command.cs@L450-L540`, `src/System.Management.Automation/engine/ScriptCommandProcessor.cs@L232-L625`) | document vs E1 code | RESOLVED | documents corrected in the same commit; both paths apply the same variant rule (CF-CMP-003) | docs-out/02-language/02.5-compiler-and-engine.md, TIERING-POLICY.md, SCRIPT-BLOCKS.md, LIFECYCLE-script-execution.md | diff --git a/docs-out/DECISIONS.md b/docs-out/DECISIONS.md index 3055263..1ef001d 100644 --- a/docs-out/DECISIONS.md +++ b/docs-out/DECISIONS.md @@ -21,3 +21,4 @@ | D-017 | `AST-CATALOG.md` children column is extracted from `InternalVisit`/`InternalVisitChildren` bodies (receivers of `.InternalVisit(`, loop variables mapped to their collection); the dataset's `ast_typed_members` is not used because it misses tuple and collection children. Statistics: 76 nodes in `ast.cs`, 9 concrete leaf-looking rows (checked: leaves or children visited through `InternalVisitChildren`). | Source reading for `IfStatementAst`, `ScriptBlockAst`, `ErrorStatementAst`, `InvokeMemberExpressionAst`. | | D-018 | T24, T25 and T27 were authored in one batch (inline, no subagents). Drafts use `[[file::Type.Member]]` tokens expanded by `tools/expand_refs.py` from `data/types.json` and `data/members-tierA.json`, so no reference range is typed by hand; unresolved tokens fail the expansion. The T27 documents own CF-CMP-001/002/003; `AlwaysCompile` is documented as unused after `rg` over `engine/`. | Rule 1 (evidence) and rule 9 (generated catalogs). | | D-019 | T26, T28, T29, T210 and T211 were authored inline (no subagents) in one batch on a branch restarted from the merged `main` (`git checkout -B … origin/main`). No reading notes existed for these areas, so each document states exactly which windows were read in its Verification line and §19; files only partly read are recorded with partial ranges in the ledger (`.work/ledger-proposals/T26_T211.csv`), not as fully read. Coverage for `Binders.cs` (8,015 lines), `LanguagePrimitives.cs` (5,888) and `MiscOps.cs` (3,765) is therefore partial; the gaps are backlog items VB-017..VB-023. | Rule 7 (read windows, no false coverage claims). | +| D-020 | The coordinator documents (pipeline map, stage contracts, call graphs, lifecycle, spec draft, quirks, design space, golden-samples index, oracle protocol, test corpus) were authored inline on a branch restarted from the merged `main`. The lifecycle trace read the host-to-engine path (`Executor`, `LocalPipeline`, `Command.CreateCommandProcessor`, `AutomationEngine.ParseScriptBlock`, `DlrScriptCommandProcessor`), which corrected an earlier statement about the execution entry (CON-013). `data/interpreter-scores.json` is a hand-authored dataset (scores and justifications with labels); totals, gates, recommendations and sensitivity are computed by `tools/gen_coord.py`. Golden samples and oracle remain empty/waived (VB-001). | Rule 1 (evidence), rule 9 (computed catalogs), prompt 2.10 scoring rules. | diff --git a/docs-out/PROGRESS.md b/docs-out/PROGRESS.md index f3df4ab..9cce03f 100644 --- a/docs-out/PROGRESS.md +++ b/docs-out/PROGRESS.md @@ -8,7 +8,7 @@ Run mode: STATIC-ONLY (`ALLOW_STATIC_ONLY=true`). Branch `claude/repo-review-exp | Preflight (section 2) | done | `.work/environment.md`, `.work/pin.json`, DECISIONS D-001..D-011 | | 0 Recon, pinning, tooling | done | datasets in `data/`, tools in `tools/`, registers, glossary, census, dependency graph, source-tree map, 297 folder cards + 40 project cards (generated; annotation overlay pending) | | 1 Repository map | done | layered-architecture, build-system, ci-cd-and-packaging, coding-standards (committed 4a6efaf) | -| 2 Language pipeline | in progress | wave 1: T21 DONE (draft); T22 DONE (draft); T23 DONE (draft); T24 T25 T27 DONE (draft); T26 T28 T29 T210 T211 DONE (draft, partial reading depth: see D-019 and VB-017..VB-023); the coordinator documents of 02-language (pipeline map, stage contracts, call graphs, lifecycle, spec draft, quirks, design space, test corpus) are next | +| 2 Language pipeline | in progress | wave 1: T21 DONE (draft); T22 DONE (draft); T23 DONE (draft); T24 T25 T27 DONE (draft); T26 T28 T29 T210 T211 DONE (draft, partial reading depth: see D-019 and VB-017..VB-023); the coordinator documents of 02-language are DONE (draft): pipeline map, stage contracts, call graphs, lifecycle, spec draft, 32 quirks, design space, golden-samples index (empty), oracle protocol (waived), test corpus; Phase 2 documents are complete except the reading-depth gaps VB-017..VB-023 | | 3 Runtime | todo | | | 4 Hosting/SDK/remoting/platform/command modules | todo | | | 5 Security | todo | | @@ -29,10 +29,12 @@ T21 tokenizer | T22 parser/grammar/errors | T23 AST | T24 semantic | T25 compile | T24 | `02-language/02.4-semantic-analysis.md` (check table generated by `tools/gen_T24.py`: 133 sites, 104 error IDs) | draft written; DEV-SEM-030, CON-009; checks pass | | T25 | `02-language/02.5-compiler-and-engine.md`, `COMPILATION-MAP.md` (`tools/gen_T25.py` + `docs-out/tools/annotations/T25-compilation-map.json`), `IR-SPEC.md` | draft written; DEV-ENG-040, CON-010, VB-015; checks pass | | T27 | `02-language/INSTRUCTION-SET.md` (`tools/gen_T27.py`), `TIERING-POLICY.md` (owns CF-CMP-001/002/003), `EQUIVALENCE-SPEC.md` | draft written; DEV-TIER-060, DEV-EQV-061, VB-013/014/016; checks pass | -| T26 | `02-language/BINDER-CATALOG.md` (`tools/gen_T26.py` + `tools/annotations/T26-binders.json`), `RUNTIME-OPERATIONS.md` | draft; 30 binders listed (10 read in depth, 20 by names); DEV-BIND-050, DEV-RTOPS-051; CON-011/012; VB-020/021 | +| T26 | `02-language/BINDER-CATALOG.md` (`tools/gen_T26.py` + `docs-out/tools/annotations/T26-binders.json`), `RUNTIME-OPERATIONS.md` | draft; 30 binders listed (10 read in depth, 20 by names); DEV-BIND-050, DEV-RTOPS-051; CON-011/012; VB-020/021 | | T28 | `02-language/02.6-types-and-operators.md` (CF-PIP-001), `OPERATOR-TABLE.md` (CF-OP-001, `tools/gen_T28.py`), `CONVERSION-RULES.md` (CF-CNV-001) | draft; DEV-TYP-070; VB-018/019 | | T29 | `02-language/SCOPE-AND-VARIABLES.md` (CF-SCP-001, `tools/gen_T29.py`), `03-runtime/execution-context-and-session-state.md` | draft; DEV-SCP-080, DEV-CTX-081; VB-017 | | T210 | `02-language/02.7-language-features.md`, `HOWTO-extend-language.md`, `SCRIPT-BLOCKS.md` | draft; DEV-SB-090, DEV-FEAT-091; VB-022 | | T211 | `02-language/02.8-language-tooling.md` | draft; DEV-TOOLS-100; VB-023 | -Next step: the coordinator documents of 02-language, then Phase 3 runtime (T31 runspace/pipeline, T32 command discovery, T33 parameter binding and cmdlet infrastructure, T34-T39) and the deeper reads listed in VB-017..VB-023. Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. +| COORD | `02-language/00-pipeline-map.md`, `STAGE-CONTRACTS.md`, `CALL-GRAPHS.md` (`tools/gen_coord.py`), `LIFECYCLE-script-execution.md`, `LANGUAGE-SPEC-DRAFT.md`, `SEMANTIC-QUIRKS.md`, `INTERPRETER-DESIGN-SPACE.md` (scores in `data/interpreter-scores.json`), `GOLDEN-SAMPLES.md`, `ORACLE-PROTOCOL.md`, `REFERENCE-TEST-CORPUS.md`, `09-synthesis/new-language-design-canvas.md`, `README.md` | draft; DEV-DESIGN-110; entry-point correction recorded (CON-013) | + +Next step: Phase 3 runtime (T31 runspace/pipeline, T32 command discovery, T33 parameter binding and cmdlet infrastructure, T34-T39) and the deeper reads listed in VB-017..VB-023. Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. diff --git a/docs-out/README.md b/docs-out/README.md new file mode 100644 index 0000000..5e47fbb --- /dev/null +++ b/docs-out/README.md @@ -0,0 +1,20 @@ +# PowerShell v7.6.9 architecture study (static-only) — index + +Pin: version `v7.6.9`, commit `523031e` (`.work/pin.json`). STATIC-ONLY run: nothing was built or executed; behavior claims are labelled `FACT-SRC` / `INFERENCE` / `UNVERIFIED` (see `VERIFICATION-BACKLOG.md`). Progress: `PROGRESS.md`. Decisions: `DECISIONS.md`. Contradictions: `CONTRADICTIONS.md`. Glossary: `GLOSSARY.md`. + +## Reading order for the language core (`02-language/`) + +1. [00-pipeline-map.md](02-language/00-pipeline-map.md) → [STAGE-CONTRACTS.md](02-language/STAGE-CONTRACTS.md) → [LIFECYCLE-script-execution.md](02-language/LIFECYCLE-script-execution.md) → [CALL-GRAPHS.md](02-language/CALL-GRAPHS.md) +2. Front end: [02.1-tokenizer.md](02-language/02.1-tokenizer.md), [TOKEN-CATALOG.md](02-language/TOKEN-CATALOG.md), [02.2-parser.md](02-language/02.2-parser.md), [GRAMMAR.ebnf.md](02-language/GRAMMAR.ebnf.md), [ERROR-CATALOG.md](02-language/ERROR-CATALOG.md), [02.3-ast.md](02-language/02.3-ast.md), [AST-CATALOG.md](02-language/AST-CATALOG.md), [02.4-semantic-analysis.md](02-language/02.4-semantic-analysis.md) +3. Back end: [02.5-compiler-and-engine.md](02-language/02.5-compiler-and-engine.md), [COMPILATION-MAP.md](02-language/COMPILATION-MAP.md), [IR-SPEC.md](02-language/IR-SPEC.md), [TIERING-POLICY.md](02-language/TIERING-POLICY.md), [INSTRUCTION-SET.md](02-language/INSTRUCTION-SET.md), [EQUIVALENCE-SPEC.md](02-language/EQUIVALENCE-SPEC.md) +4. Semantics: [02.6-types-and-operators.md](02-language/02.6-types-and-operators.md), [OPERATOR-TABLE.md](02-language/OPERATOR-TABLE.md), [CONVERSION-RULES.md](02-language/CONVERSION-RULES.md), [BINDER-CATALOG.md](02-language/BINDER-CATALOG.md), [RUNTIME-OPERATIONS.md](02-language/RUNTIME-OPERATIONS.md), [SCOPE-AND-VARIABLES.md](02-language/SCOPE-AND-VARIABLES.md), [SCRIPT-BLOCKS.md](02-language/SCRIPT-BLOCKS.md), [execution-context-and-session-state.md](03-runtime/execution-context-and-session-state.md) +5. Synthesis: [LANGUAGE-SPEC-DRAFT.md](02-language/LANGUAGE-SPEC-DRAFT.md), [SEMANTIC-QUIRKS.md](02-language/SEMANTIC-QUIRKS.md), [INTERPRETER-DESIGN-SPACE.md](02-language/INTERPRETER-DESIGN-SPACE.md), [02.7-language-features.md](02-language/02.7-language-features.md), [HOWTO-extend-language.md](02-language/HOWTO-extend-language.md), [02.8-language-tooling.md](02-language/02.8-language-tooling.md), [new-language-design-canvas.md](09-synthesis/new-language-design-canvas.md) +6. Evidence and verification: [REFERENCE-TEST-CORPUS.md](02-language/REFERENCE-TEST-CORPUS.md), [GOLDEN-SAMPLES.md](02-language/GOLDEN-SAMPLES.md) (empty), [ORACLE-PROTOCOL.md](02-language/ORACLE-PROTOCOL.md) (waived) + +## Repository overview (`00-overview/`, `01-repository/`, `10-command-modules/`) + +[repo-census](00-overview/repo-census.md), [source-tree-map](00-overview/source-tree-map.md), [project-dependency-graph](00-overview/project-dependency-graph.md), [external-dependencies](00-overview/external-dependencies.md), [layered-architecture](00-overview/layered-architecture.md), [build-system](01-repository/build-system.md), [ci-cd-and-packaging](01-repository/ci-cd-and-packaging.md), [coding-standards](01-repository/coding-standards.md), [command module families](10-command-modules/_index.md). + +## Data and tools + +`data/` holds the extracted datasets (all `APPROX`, see `data/README.md`); `tools/` holds the extraction and generator scripts (`gen_T21.py` … `gen_coord.py`, `expand_refs.py`, `consist.py`, `numbers.py`, `coordinator.py`); `.work/` holds notes, claims, ledgers and per-task proposals. diff --git a/docs-out/data/canonical-facts.json b/docs-out/data/canonical-facts.json index fb987c1..ab4545b 100644 --- a/docs-out/data/canonical-facts.json +++ b/docs-out/data/canonical-facts.json @@ -41,13 +41,14 @@ { "id": "CF-CMP-003", "subject": "optimized-versus-unoptimized preconditions", - "statement": "**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used for script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables.", + "statement": "**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used by the command processors that run script blocks (`DlrScriptCommandProcessor` constructor) and script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables.", "label": "FACT-SRC", "evidence": [ "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83", "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177", "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552", - "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847" + "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847", + "src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.Init@L274-L296" ], "owner": "02-language/TIERING-POLICY.md", "status": "open", diff --git a/docs-out/data/claims/COORD.jsonl b/docs-out/data/claims/COORD.jsonl new file mode 100644 index 0000000..8074b42 --- /dev/null +++ b/docs-out/data/claims/COORD.jsonl @@ -0,0 +1,8 @@ +{"id": "CLM-LIFE-0001", "subject_key": "lifecycle.script_parse_path", "predicate": "script command text is parsed by", "value": "AutomationEngine.ParseScriptBlock via EngineParser.Parse (no script block cache)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0002", "subject_key": "lifecycle.incomplete_input", "predicate": "a parse error that is incomplete input raises", "value": "IncompleteParseException", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0003", "subject_key": "lifecycle.nolanguage_scripts", "predicate": "script text with NoLanguage and runspace origin", "value": "rejected (ScriptsNotAllowed)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0004", "subject_key": "lifecycle.cmdletbinding_processor", "predicate": "script block using cmdlet binding runs in", "value": "CommandProcessor over a FunctionInfo", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0005", "subject_key": "lifecycle.script_processor", "predicate": "other script blocks run in", "value": "DlrScriptCommandProcessor", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor@L232-L625"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0006", "subject_key": "lifecycle.flowcontrol_across_commands", "predicate": "FlowControlException in a clause", "value": "rethrown out of the command processor", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L625"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0007", "subject_key": "lifecycle.exit_script_file", "predicate": "ExitException from a script file", "value": "sets LASTEXITCODE and marks pipeline failed when non-zero", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L625"], "doc": "LANG-LIFECYCLE-001", "cf": null} +{"id": "CLM-LIFE-0008", "subject_key": "lifecycle.outdefault_appended", "predicate": "executor appends Out-Default for rendering when", "value": "AddOutputter option set", "label": "FACT-SRC", "evidence": ["src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor.ExecuteCommandHelper@L335-L357"], "doc": "LANG-LIFECYCLE-001", "cf": null} diff --git a/docs-out/data/interpreter-scores.json b/docs-out/data/interpreter-scores.json index 0637a08..ddf2d1a 100644 --- a/docs-out/data/interpreter-scores.json +++ b/docs-out/data/interpreter-scores.json @@ -1 +1,472 @@ -[] \ No newline at end of file +{ + "_note": "Hand-authored scores for INTERPRETER-DESIGN-SPACE.md (task coordinator). Scores 1-5; weights sum to 100; totals and recommendations are computed by tools/gen_coord.py, never typed.", + "weights": { + "fidelity": 25, + "sandbox": 15, + "startup": 10, + "throughput": 10, + "debug": 10, + "aot": 10, + "determinism": 5, + "memory": 5, + "observability": 5, + "complexity": 5 + }, + "options": [ + { + "id": "S0", + "name": "PowerShell-faithful tiered pipeline (AST \u2192 expression trees \u2192 light interpreter \u2192 JIT)", + "scores": { + "fidelity": 5, + "sandbox": 3, + "startup": 3, + "throughput": 5, + "debug": 4, + "aot": 1, + "determinism": 2, + "memory": 2, + "observability": 4, + "complexity": 1 + }, + "why": { + "fidelity": [ + "FACT-SRC", + "It is the strategy PowerShell itself uses, so observable behavior is reproduced by construction (TIERING-POLICY, EQUIVALENCE-SPEC)." + ], + "sandbox": [ + "INFERENCE", + "Expression trees and call sites run arbitrary delegates; hooks exist at binders and runtime helpers but the JIT tier bypasses interpreter-level checks." + ], + "startup": [ + "FACT-SRC", + "The interpreter tier starts cheaply but the compiler front end builds trees for every block (COMPILATION-MAP)." + ], + "throughput": [ + "FACT-SRC", + "Hot blocks and loops are JIT-compiled after 32 calls or 16 iterations (TIERING-POLICY)." + ], + "debug": [ + "FACT-SRC", + "Sequence points and debugger hooks are emitted in both tiers (IR-SPEC \u00a75)." + ], + "aot": [ + "EXTERNAL", + "Expression.Compile and Reflection.Emit need a JIT; under NativeAOT trees fall back to a slow interpreter." + ], + "determinism": [ + "FACT-SRC", + "Asynchronous promotion makes the executing tier nondeterministic (TIERING-POLICY \u00a77.5)." + ], + "memory": [ + "FACT-SRC", + "Trees, instruction arrays, call sites and caches are retained per block." + ], + "observability": [ + "FACT-SRC", + "ETW compile events and script block logging exist (02.5 \u00a714)." + ], + "complexity": [ + "FACT-SRC", + "About 6,600 lines of compiler plus the interpreter, binders and DLR dependency." + ] + }, + "gates": { + "H1": true, + "H2_jit": true, + "H2_aot": false, + "H3": true, + "note": "AOT: JIT tier and Expression.Compile unavailable (EXTERNAL)" + } + }, + { + "id": "S1", + "name": "AST tree-walking interpreter", + "scores": { + "fidelity": 4, + "sandbox": 5, + "startup": 5, + "throughput": 2, + "debug": 5, + "aot": 5, + "determinism": 5, + "memory": 4, + "observability": 4, + "complexity": 5 + }, + "why": { + "fidelity": [ + "INFERENCE", + "Every protocol of IR-SPEC \u00a75 can be implemented once in the evaluator; no PowerShell-visible behavior requires a compiled form." + ], + "sandbox": [ + "INFERENCE", + "All execution passes through one evaluator, so policy checks sit at every node visit and call." + ], + "startup": [ + "INFERENCE", + "No compile step: execution starts at the first node." + ], + "throughput": [ + "INFERENCE", + "Each node costs a virtual dispatch and allocations for values; no slot access without a resolver pass." + ], + "debug": [ + "INFERENCE", + "The evaluator is the debugger: breakpoints and stepping are plain hooks on statements." + ], + "aot": [ + "EXTERNAL", + "Pure managed code without dynamic code generation runs under NativeAOT." + ], + "determinism": [ + "INFERENCE", + "Single tier: results never depend on promotion timing." + ], + "memory": [ + "INFERENCE", + "Only the AST is retained; no trees or instruction arrays." + ], + "observability": [ + "INFERENCE", + "Trace of statements and calls is trivial to emit." + ], + "complexity": [ + "INFERENCE", + "Smallest implementation; the main work is the language rules, not an engine." + ] + }, + "gates": { + "H1": true, + "H2_jit": true, + "H2_aot": true, + "H3": true, + "note": "Fidelity depends on implementing the protocols of IR-SPEC \u00a75 and the equivalence obligations EQ-01..EQ-08" + } + }, + { + "id": "S2", + "name": "Custom bytecode VM", + "scores": { + "fidelity": 4, + "sandbox": 5, + "startup": 4, + "throughput": 3, + "debug": 3, + "aot": 5, + "determinism": 5, + "memory": 4, + "observability": 3, + "complexity": 2 + }, + "why": { + "fidelity": [ + "INFERENCE", + "A VM can implement the same protocols; the extra translation step adds risk of divergence from the AST semantics." + ], + "sandbox": [ + "INFERENCE", + "A VM executes only its own instructions, so hooks go into the instruction loop." + ], + "startup": [ + "INFERENCE", + "A compile step to bytecode is cheap compared with trees but not free." + ], + "throughput": [ + "INFERENCE", + "A tight instruction loop beats tree-walking (PowerShell's own interpreter is of this kind, INSTRUCTION-SET)." + ], + "debug": [ + "INFERENCE", + "Needs source maps from instructions to sequence points." + ], + "aot": [ + "EXTERNAL", + "Managed bytecode interpreter has no AOT constraints." + ], + "determinism": [ + "INFERENCE", + "Single tier, deterministic." + ], + "memory": [ + "INFERENCE", + "Instruction arrays and constant pools per block." + ], + "observability": [ + "INFERENCE", + "Instruction-level tracing is possible but noisy." + ], + "complexity": [ + "INFERENCE", + "A compiler, an instruction set and a frame model must be designed and tested." + ] + }, + "gates": { + "H1": true, + "H2_jit": true, + "H2_aot": true, + "H3": true, + "note": "Must reproduce flow-control and error-action protocols in the instruction set" + } + }, + { + "id": "S3", + "name": "System.Linq.Expressions compilation to delegates", + "scores": { + "fidelity": 5, + "sandbox": 3, + "startup": 2, + "throughput": 5, + "debug": 3, + "aot": 2, + "determinism": 3, + "memory": 2, + "observability": 3, + "complexity": 3 + }, + "why": { + "fidelity": [ + "FACT-SRC", + "PowerShell compiles to expression trees; behavior is reproduced via the same IR (COMPILATION-MAP)." + ], + "sandbox": [ + "INFERENCE", + "Compiled delegates run unchecked CLR code once produced; policy must be emitted into the trees." + ], + "startup": [ + "FACT-SRC", + "Tree building and Expression.Compile cost start-up time; PowerShell avoids it by interpreting first." + ], + "throughput": [ + "FACT-SRC", + "JIT-compiled delegates are the fast path of PowerShell hot code." + ], + "debug": [ + "INFERENCE", + "Debug info is possible (DebugInfo nodes) but stepping compiled code is harder." + ], + "aot": [ + "EXTERNAL", + "Expression.Compile under NativeAOT degrades to an interpreter." + ], + "determinism": [ + "INFERENCE", + "Compile on demand with caching is deterministic if synchronous." + ], + "memory": [ + "FACT-SRC", + "Delegates and trees are cached per block (CompiledScriptBlockData)." + ], + "observability": [ + "INFERENCE", + "Compile events can be logged." + ], + "complexity": [ + "FACT-SRC", + "Needs the compiler, binders and caches but not the light interpreter." + ] + }, + "gates": { + "H1": true, + "H2_jit": true, + "H2_aot": false, + "H3": true, + "note": "AOT: Expression.Compile falls back to interpretation (EXTERNAL)" + } + }, + { + "id": "S4", + "name": "Reflection.Emit dynamic assemblies", + "scores": { + "fidelity": 3, + "sandbox": 2, + "startup": 1, + "throughput": 5, + "debug": 2, + "aot": 1, + "determinism": 3, + "memory": 1, + "observability": 2, + "complexity": 1 + }, + "why": { + "fidelity": [ + "FACT-SRC", + "PowerShell uses Reflection.Emit only to define classes and enums (TypeDefiner), not for the dynamic language core." + ], + "sandbox": [ + "INFERENCE", + "Emitted code calls the CLR directly; checks must be emitted explicitly." + ], + "startup": [ + "EXTERNAL", + "Emitting and loading assemblies costs the most start-up time of the options." + ], + "throughput": [ + "EXTERNAL", + "Emitted IL is JIT-compiled to native code." + ], + "debug": [ + "INFERENCE", + "Debugging emitted code needs symbol emission." + ], + "aot": [ + "EXTERNAL", + "Reflection.Emit is unavailable under NativeAOT." + ], + "determinism": [ + "INFERENCE", + "Deterministic if emission is synchronous." + ], + "memory": [ + "EXTERNAL", + "Dynamic assemblies stay loaded unless collectible (AssemblyLoadContext)." + ], + "observability": [ + "INFERENCE", + "Emitted code is opaque to tracing." + ], + "complexity": [ + "INFERENCE", + "Emitting IL for dynamic scoping, pipelines and error-action semantics is the most complex option." + ] + }, + "gates": { + "H1": false, + "H2_jit": true, + "H2_aot": false, + "H3": false, + "note": "No evidence it can host the dynamic-scoping core (INFERENCE); emitted code bypasses evaluator hooks" + } + }, + { + "id": "S5", + "name": "Source-generated ahead-of-time compilation (built-in commands)", + "scores": { + "fidelity": 2, + "sandbox": 4, + "startup": 5, + "throughput": 5, + "debug": 2, + "aot": 5, + "determinism": 5, + "memory": 5, + "observability": 3, + "complexity": 3 + }, + "why": { + "fidelity": [ + "INFERENCE", + "It cannot run user scripts; it only compiles the shell's own built-in commands." + ], + "sandbox": [ + "INFERENCE", + "Generated code is reviewed at build time and calls only allowed APIs." + ], + "startup": [ + "EXTERNAL", + "AOT or R2R code needs no JIT at start-up." + ], + "throughput": [ + "EXTERNAL", + "Native code is the fastest." + ], + "debug": [ + "INFERENCE", + "Generated code is debuggable like normal C#." + ], + "aot": [ + "EXTERNAL", + "Fully compatible with NativeAOT." + ], + "determinism": [ + "INFERENCE", + "Deterministic." + ], + "memory": [ + "EXTERNAL", + "No runtime code generation memory." + ], + "observability": [ + "INFERENCE", + "Generated hooks can add tracing." + ], + "complexity": [ + "INFERENCE", + "A source generator is extra work but each command stays plain code." + ] + }, + "gates": { + "H1": false, + "H2_jit": true, + "H2_aot": true, + "H3": true, + "note": "Complement for built-in commands, not a strategy for user scripts" + } + }, + { + "id": "S6", + "name": "Hybrid: tree-walker plus bytecode tier for hot paths", + "scores": { + "fidelity": 4, + "sandbox": 5, + "startup": 4, + "throughput": 4, + "debug": 3, + "aot": 5, + "determinism": 4, + "memory": 3, + "observability": 3, + "complexity": 2 + }, + "why": { + "fidelity": [ + "INFERENCE", + "Behavior is defined by the tree-walker; the bytecode tier must match it (EQUIVALENCE-SPEC applies)." + ], + "sandbox": [ + "INFERENCE", + "Both tiers are interpreters, so hooks are placed in two loops." + ], + "startup": [ + "INFERENCE", + "Starts as a tree-walker; bytecode is generated only for hot code." + ], + "throughput": [ + "INFERENCE", + "Hot paths run the bytecode loop." + ], + "debug": [ + "INFERENCE", + "Two tiers need two debug paths." + ], + "aot": [ + "EXTERNAL", + "Managed code only; no JIT requirement." + ], + "determinism": [ + "INFERENCE", + "Deterministic if promotion is synchronous and count-based." + ], + "memory": [ + "INFERENCE", + "AST plus bytecode for hot blocks." + ], + "observability": [ + "INFERENCE", + "Promotion events can be traced." + ], + "complexity": [ + "INFERENCE", + "Two engines and an equivalence test harness." + ] + }, + "gates": { + "H1": true, + "H2_jit": true, + "H2_aot": true, + "H3": true, + "note": "Build only after S1 is verified; triggers in INTERPRETER-DESIGN-SPACE \u00a77" + } + } + ] +} \ No newline at end of file diff --git a/docs-out/data/lessons.json b/docs-out/data/lessons.json index d61cced..36e374d 100644 --- a/docs-out/data/lessons.json +++ b/docs-out/data/lessons.json @@ -741,12 +741,34 @@ "verdict": "AVOID", "area": "NLD-RUN", "text": "Do not port COM binders and `PSGetMemberBinder`'s COM/ETS-adapter paths in the first versions. Replace with a plain reflection member lookup.", - "evidence": "`src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComBinder@L1-L400`.", + "evidence": "`src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComBinder@L1-L378`.", "dev": "DEV-BIND-050", "doc": "LANG-BINDERCAT-001", "file": "02-language/BINDER-CATALOG.md", "line": 80 }, + { + "id": "LSN-CALLG-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Keep the five entry points (tokenize, parse, check, compile, invoke) as separate callable stages with narrow signatures so each can be tested alone.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`.", + "dev": null, + "doc": "LANG-CALLG-001", + "file": "02-language/CALL-GRAPHS.md", + "line": 352 + }, + { + "id": "LSN-CALLG-02", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Generate such graphs in CI from the shell's own sources to catch accidental coupling between stages. Why: the PowerShell graphs show the compiler calling parser-time helpers and the AST calling the compiler.", + "evidence": "`src/System.Management.Automation/engine/parser/ast.cs::VariableExpressionAst@L9404-L9624`.", + "dev": null, + "doc": "LANG-CALLG-001", + "file": "02-language/CALL-GRAPHS.md", + "line": 353 + }, { "id": "LSN-CONV-01", "verdict": "REPLICATE", @@ -828,13 +850,24 @@ "id": "LSN-EQV-03", "verdict": "AVOID", "area": "NLD-RUN", - "text": "Do not port the typed arithmetic/comparison instruction families or the DLR-derived interpreter; they are unreachable or defective for PowerShell. Replace with the evaluator's own numeric semantics from CONVERSION-RULES (planned T28).", + "text": "Do not port the typed arithmetic/comparison instruction families or the DLR-derived interpreter; they are unreachable or defective for PowerShell. Replace with the evaluator's own numeric semantics from [CONVERSION-RULES.md](CONVERSION-RULES.md).", "evidence": "src/System.Management.Automation/engine/interpreter/Instruction.cs::Instruction@L27-L70.", "dev": "DEV-EQV-061", "doc": "LANG-EQUIV-001", "file": "02-language/EQUIVALENCE-SPEC.md", "line": 126 }, + { + "id": "LSN-GOLD-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Keep an oracle-recorded conformance corpus as the acceptance test of the new interpreter.", + "evidence": "`test/powershell/Language/Parser/` (the repository's own language tests, see REFERENCE-TEST-CORPUS).", + "dev": null, + "doc": "LANG-GOLDEN-001", + "file": "02-language/GOLDEN-SAMPLES.md", + "line": 91 + }, { "id": "LSN-HOWTO-01", "verdict": "ADAPT", @@ -868,6 +901,39 @@ "file": "02-language/HOWTO-extend-language.md", "line": 145 }, + { + "id": "LSN-DESIGN-01", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Build `S1`, a tree-walking evaluator that implements the protocols of IR-SPEC \u00a75, as the first and only engine until measurements justify another tier. Why: same observable behavior as PowerShell's compiled pipeline with a fraction of the machinery.", + "evidence": "`src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661`.", + "dev": null, + "doc": "LANG-DESIGNSPACE-001", + "file": "02-language/INTERPRETER-DESIGN-SPACE.md", + "line": 228 + }, + { + "id": "LSN-DESIGN-02", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "If a second tier is added, test it differentially against the first on the whole corpus.", + "evidence": "`src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda@L43-L303`.", + "dev": null, + "doc": "LANG-DESIGNSPACE-001", + "file": "02-language/INTERPRETER-DESIGN-SPACE.md", + "line": 229 + }, + { + "id": "LSN-DESIGN-03", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not adopt `Reflection.Emit` or `Expression.Compile` as the core engine.", + "evidence": "`src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505`.", + "dev": "DEV-DESIGN-110", + "doc": "LANG-DESIGNSPACE-001", + "file": "02-language/INTERPRETER-DESIGN-SPACE.md", + "line": 230 + }, { "id": "LSN-IR-01", "verdict": "ADAPT", @@ -923,6 +989,72 @@ "file": "02-language/IR-SPEC.md", "line": 186 }, + { + "id": "LSN-SPEC-01", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Treat this specification plus the quirks catalog as the acceptance baseline and add oracle-confirmed tests per statement.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8191`.", + "dev": null, + "doc": "LANG-SPEC-001", + "file": "02-language/LANGUAGE-SPEC-DRAFT.md", + "line": 120 + }, + { + "id": "LSN-SPEC-02", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Keep the specification in the shell's repository as executable examples (snippet + expected result) rather than prose only. Why: same content, checkable.", + "evidence": "`test/powershell/Language/Parser/` (test style).", + "dev": null, + "doc": "LANG-SPEC-001", + "file": "02-language/LANGUAGE-SPEC-DRAFT.md", + "line": 121 + }, + { + "id": "LSN-LIFE-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Treat the command line as one script command followed by an output command in an ordinary pipeline, so interactive and file execution share a path.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor.ExecuteCommandHelper@L335-L357.", + "dev": null, + "doc": "LANG-LIFECYCLE-001", + "file": "02-language/LIFECYCLE-script-execution.md", + "line": 148 + }, + { + "id": "LSN-LIFE-02", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Parse the whole text before running anything and report incomplete input distinctly so a REPL can ask for another line.", + "evidence": "src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine.ParseScriptBlock@L96-L127.", + "dev": null, + "doc": "LANG-LIFECYCLE-001", + "file": "02-language/LIFECYCLE-script-execution.md", + "line": 149 + }, + { + "id": "LSN-LIFE-03", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Run script blocks through a begin/process/end command processor that sets `$_`/`$input`, scopes, error pipe and language mode around each clause.", + "evidence": "src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor.RunClause@L485-L623.", + "dev": null, + "doc": "LANG-LIFECYCLE-001", + "file": "02-language/LIFECYCLE-script-execution.md", + "line": 150 + }, + { + "id": "LSN-LIFE-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Merge the `CommandProcessor`/`DlrScriptCommandProcessor` split into one script command processor. Why: the split exists for cmdlet-style binding of advanced functions; the observable begin/process/end behavior is the same.", + "evidence": "src/System.Management.Automation/engine/hostifaces/Command.cs::Command.CreateCommandProcessor@L450-L540.", + "dev": null, + "doc": "LANG-LIFECYCLE-001", + "file": "02-language/LIFECYCLE-script-execution.md", + "line": 151 + }, { "id": "LSN-OP-01", "verdict": "REPLICATE", @@ -956,6 +1088,39 @@ "file": "02-language/OPERATOR-TABLE.md", "line": 136 }, + { + "id": "LSN-ORACLE-01", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Use the recorded oracle outputs as the expected values of the new interpreter's conformance suite and re-run the oracle for each new quirk. Why: same coverage with an automated diff instead of manual review.", + "evidence": "`test/powershell/Language/Operators/` (existing assertion style).", + "dev": null, + "doc": "LANG-ORACLE-001", + "file": "02-language/ORACLE-PROTOCOL.md", + "line": 94 + }, + { + "id": "LSN-CORPUS-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Organize the new shell's specification tests by the same feature folders (parser, operators, scripting, classes) and keep one behavior per test.", + "evidence": "`test/powershell/Language/Parser/` (folder layout in `data/tests.json`).", + "dev": null, + "doc": "LANG-TESTCORPUS-001", + "file": "02-language/REFERENCE-TEST-CORPUS.md", + "line": 234 + }, + { + "id": "LSN-CORPUS-02", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Write the new suite in the shell's own language plus a small C# harness, so tests run without PowerShell. Why: same coverage, no dependency on the reference implementation.", + "evidence": "`test/xUnit/csharp/test_Binders.cs` (xUnit tests of internal binders).", + "dev": null, + "doc": "LANG-TESTCORPUS-001", + "file": "02-language/REFERENCE-TEST-CORPUS.md", + "line": 235 + }, { "id": "LSN-RTOPS-01", "verdict": "REPLICATE", @@ -1143,6 +1308,72 @@ "file": "02-language/SCRIPT-BLOCKS.md", "line": 111 }, + { + "id": "LSN-QRK-01", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Treat QRK-01 to QRK-27 and QRK-30 to QRK-32 as part of the baseline language and specify them with tests before implementing.", + "evidence": "`src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`.", + "dev": null, + "doc": "LANG-QUIRKS-001", + "file": "02-language/SEMANTIC-QUIRKS.md", + "line": 124 + }, + { + "id": "LSN-QRK-02", + "verdict": "ADAPT", + "area": "NLD-OP", + "text": "Keep the observable behavior of QRK-04, QRK-21 and QRK-22 with safer mechanisms (full scan, single comparer, caps in all modes).", + "evidence": "`src/System.Management.Automation/engine/runtime/Operations/StringOps.cs::StringOps@L15-L107`.", + "dev": null, + "doc": "LANG-QUIRKS-001", + "file": "02-language/SEMANTIC-QUIRKS.md", + "line": 125 + }, + { + "id": "LSN-QRK-03", + "verdict": "AVOID", + "area": "NLD-GRM", + "text": "Do not reproduce parse-time side effects (QRK-28, QRK-29) or tokenizer-level pragma handling (QRK-03).", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168`.", + "dev": "DEV-GRM-010", + "doc": "LANG-QUIRKS-001", + "file": "02-language/SEMANTIC-QUIRKS.md", + "line": 126 + }, + { + "id": "LSN-STAGES-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Keep the six boundaries as explicit interfaces with the three error channels (diagnostics, runtime exception, error record).", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`.", + "dev": null, + "doc": "LANG-STAGES-001", + "file": "02-language/STAGE-CONTRACTS.md", + "line": 172 + }, + { + "id": "LSN-STAGES-02", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Place static policy at B3, content policy at B4, and a single invoke/convert choke point at B5.", + "evidence": "`src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.PerformSecurityChecks@L197-L283`.", + "dev": null, + "doc": "LANG-STAGES-001", + "file": "02-language/STAGE-CONTRACTS.md", + "line": 173 + }, + { + "id": "LSN-STAGES-03", + "verdict": "ADAPT", + "area": "NLD-GRM", + "text": "Pass tokenizer mode as an argument and keep annotations in a side model. Why: same behavior, removes hidden state.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`.", + "dev": null, + "doc": "LANG-STAGES-001", + "file": "02-language/STAGE-CONTRACTS.md", + "line": 174 + }, { "id": "LSN-TIER-01", "verdict": "REPLICATE", diff --git a/docs-out/data/numbers-extra.json b/docs-out/data/numbers-extra.json index ea2b2b7..a1ac546 100644 --- a/docs-out/data/numbers-extra.json +++ b/docs-out/data/numbers-extra.json @@ -180,5 +180,68 @@ "dataset": "members-tierA.json", "label": "APPROX", "notes": "T29" + }, + { + "key": "PARSER_RULE_NODES", + "cmd": "python3 docs-out/tools/gen_coord.py --count callgraph_nodes", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_TOTAL", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_total", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_REPLICATE", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_replicate", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_ADAPT", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_adapt", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_AVOID", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_avoid", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_REDESIGN", + "cmd": "python3 docs-out/tools/gen_coord.py --count quirks_redesign", + "dataset": "SEMANTIC-QUIRKS.md", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "LANGUAGE_TEST_FILES", + "cmd": "python3 docs-out/tools/gen_coord.py --count test_files", + "dataset": "tests.json", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "LANGUAGE_TEST_ITS", + "cmd": "python3 docs-out/tools/gen_coord.py --count test_its", + "dataset": "tests.json", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "INTERPRETER_OPTIONS", + "cmd": "python3 docs-out/tools/gen_coord.py --count options", + "dataset": "interpreter-scores.json", + "label": "APPROX", + "notes": "coordinator documents" } ] \ No newline at end of file diff --git a/docs-out/data/numbers.json b/docs-out/data/numbers.json index 8bb5f48..6481242 100644 --- a/docs-out/data/numbers.json +++ b/docs-out/data/numbers.json @@ -601,5 +601,86 @@ "evidence": "E1", "label": "APPROX", "notes": "T29" + }, + { + "key": "PARSER_RULE_NODES", + "value": 62, + "dataset": "members-tierA.json", + "command": "python3 docs-out/tools/gen_coord.py --count callgraph_nodes", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_TOTAL", + "value": 32, + "dataset": "SEMANTIC-QUIRKS.md", + "command": "python3 docs-out/tools/gen_coord.py --count quirks_total", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_REPLICATE", + "value": 26, + "dataset": "SEMANTIC-QUIRKS.md", + "command": "python3 docs-out/tools/gen_coord.py --count quirks_replicate", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_ADAPT", + "value": 3, + "dataset": "SEMANTIC-QUIRKS.md", + "command": "python3 docs-out/tools/gen_coord.py --count quirks_adapt", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_AVOID", + "value": 3, + "dataset": "SEMANTIC-QUIRKS.md", + "command": "python3 docs-out/tools/gen_coord.py --count quirks_avoid", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "QUIRKS_REDESIGN", + "value": 0, + "dataset": "SEMANTIC-QUIRKS.md", + "command": "python3 docs-out/tools/gen_coord.py --count quirks_redesign", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "LANGUAGE_TEST_FILES", + "value": 81, + "dataset": "tests.json", + "command": "python3 docs-out/tools/gen_coord.py --count test_files", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "LANGUAGE_TEST_ITS", + "value": 1249, + "dataset": "tests.json", + "command": "python3 docs-out/tools/gen_coord.py --count test_its", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" + }, + { + "key": "INTERPRETER_OPTIONS", + "value": 7, + "dataset": "interpreter-scores.json", + "command": "python3 docs-out/tools/gen_coord.py --count options", + "evidence": "E1", + "label": "APPROX", + "notes": "coordinator documents" } ] \ No newline at end of file diff --git a/docs-out/data/subject-keys.json b/docs-out/data/subject-keys.json index 24412dc..8386b4e 100644 --- a/docs-out/data/subject-keys.json +++ b/docs-out/data/subject-keys.json @@ -37,6 +37,14 @@ "hashtable.add.comparer": "", "hashtable.literal.duplicate": "", "interpreter.max_parameters": "", + "lifecycle.cmdletbinding_processor": "", + "lifecycle.exit_script_file": "", + "lifecycle.flowcontrol_across_commands": "", + "lifecycle.incomplete_input": "", + "lifecycle.nolanguage_scripts": "", + "lifecycle.outdefault_appended": "", + "lifecycle.script_parse_path": "", + "lifecycle.script_processor": "", "loopflow.empty_label": "", "operator.precedence.levels": "", "output.unroll_depth": "", diff --git a/docs-out/tools/__pycache__/gen_coord.cpython-313.pyc b/docs-out/tools/__pycache__/gen_coord.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..960860a331803fa61f62d1013c094927613cd2f3 GIT binary patch literal 30895 zcmdVDdw3gHb|+W`6hS=reu<(UE{b{(G(n1@p47vlNJ^s2CkoJu5=jsQDM%zhRe_QS z(>9%Cx5=mzQMNlxb$Zj(q-UB+dM0$IGohz56L~gYT28Xl1%sBfMdK-|p=1+^PaVigLRre{D&;b?e?+x9&an+;h)4_neDo+1YjuS0VquU;g3*$NeXI zFfNB4dG;T8j{62DaWW_IlIc}m=GomOo7mkfo7r8E1>DWAW}LFfmQ$H>rpR3|x$}(b zQg?S@DsIG3vdD<1?(TJ|xDk^CDPxaGd)kAR(IscS;7c2{{PgfbmSi#BmverN_j07n zWn^-`I%LCUYO%NwPQD7Q)4gmAO88R1LPO9(|tL|A@d zi&Wllmg9V-9Ctx*A5ZV07T>~MDBr^E(K8H4iQKSh*yOD6-4i%&I1uXdhr{w<&DezK z67d+2eUZSRIOdZhiWnXeBbNdqD-jI&B4Jq^4Ex7N10gMQ=PuWXFEl*v8xDw-^7u$V zJm9Rc+q=f)v9J;l8^p>wr`Xik*5*3lZtOZK)r<~``sEVa1M;XZI4JskBO~Il>>InJ zh(q{7Bzz?h3PuCf)CdJPe|U5(ID$wpbPai8Wym)ga1HpBz+ko5*wy9kJnKAYH!2pz z%3V&ewZqfm?sB&vbTzj~ttUEMQdeVB3$?%{%7NivI3yk_cezfqbTlabupCfat|Q0@ z4F+WNe@G6Gii5t0uQnKp1mrO}fY7BCthua&Lk(#kq)%TD1cxsX2qDeXCtgfsgAHD)^@S7%@{S7NM9TZkD%uwzAFv^*egH2 z+bQlm;8G?+zy>9#)P@4rlLPM>MAC4`<@bfI`4kPl%9X%KAOZ->;p>WcJs7zpjsz4y z4P!Y@xRYv?maedN)!17@W8=sM<}lcg21W-0vf_#a19BrPQmth$%#dB9fykxsp!Tw! z^Mc{hb5k&=P8&b=t@MUq#DMlncwF`e7&sK5cfdF33Wr7}su`9ttPR|V$UZ-fC(yog zJi;1}wuk+SD?A>lO`9LJtEK?~T0=wSVA=~`8OnC|c!;5VgE%$;D1_=$wM4?<5v6v; z-0pF?80gt-W0S2f6voI+5=8oF#Pw(}qD+Rz7>ZuBxA_JFBlt>#KF!2Rt=ogc0Y&u5 z0ddq9@dG7@Q20b`v_zVngia&j>l!lesN8|6s{{uFK(8<$E9*sfDB?&k6cA-#V*n5| zu(CnyVqFsJ#mZ*a&YioQ_D_h4bvR9l%qGI1fShm)28Mj&Bay+NKa#MyTe>>?+?}1C z1YZ-i8Uv6JXlSFAitMkICX`5Ew5Ht`0<|IvR?|2h38MoTz1l!%7!9fQ>#GSPnO1~m z{5|_E1mEC#SRw^(eeFpmAhLz)$uORKEJn;tb*#GF#;Tf@-jiiKcsXN*G3d2%-^$(N zJDiybfv8VH7zz6Z6GC__5K5TBNiyC-{&;{Sgy&wU@%9f!c8; zAlG(bRAlK=U}U7$fKu)A_>$mH$+Sjq#GkSa!4&tzk$daFZy$Kq^X)C)*>PvbcdGAH ze>bww9WSem7w?Qa>Qtfb847ZKf|+q;BEn#}8!hQI0g6codl@frz~oLZufJ!kSs`-| z7Afs=C!uPj5Y#2Jmp{bkaFU?kGwL(a-dO|w27E(;S6FK-yk-O9mHLMUvpKE@f7&Hk z&gSjW+l;kzWsfq64?19W=NhF`cYG~B^<>1nrro?`1&?6GcUSQ=RJ+sG@kPdXWXQGu z9-puVLjLezFf<&sRT^tobRE_$=Y`NKHC3-5Wa+iEs6_?Ji00^P-Dprz@MeO)z5>Y! zyfo0yq6m)g-{8EQJ2^x~BRLF}0TjvPLb9~xuz8ZDv@#UFpuQxKsmxZ+DRg{7pH1-g zPY4Q_=o!OuczjGINaBefO$fA@D3mKs$puu{dT7KqIxy%vk|+TM8>u-o5(a%zjv%6? zZ$~NR5Q2Z4;+A}WIsE?cecz8RKiC~R(y6*nKXUiP+&$`r{3W zx+2G}$f^>FUm5@C$hb^wsnaIY97Q9r3?xi)Ad#uFB?(?iSd@TI_Fqb5jLBGk5*gR! zU?h+*E8_#s3^|A1Ge>;GvPdyLk}zYkDnu?sv0oImz6jI)xszc0!@v~kC?>m^?z>Hh2^8xy zwPOq&=&fQ3aaYsG>sF*_sY9GIqa#}IJQt84Xay3sq4ANCK3Z=QnR+3hinULVGYaP9 z<(Kd|jS@0Y%e(N*0)n>EgkXyMMP_a+bKR2@C)L)A%O`r{g?%5L=)GS)d+qkQd4B%o zyE$r6?PC7of%i6kRCG9YqBmC9H*J05D4gp2{Hd8M>gN@jN#9tvM_57sk^p>4xd20I z6*&le_jRR%F^z7mLF?Q$RI%#nVX_+~rq&HcflTh~RYmb=P}Q}lla!p0sTO)Uj4}pp z%`Ydb_i}82c}Z}gHC~gz8KeT!8pCH})XN1PQ6^bVrX@utsmy6j2z^+Y*~F5my=CBVMIf3a%wVJyLO^gQ`UDyX4^GGpNTPl!7Z6Nwk24+b z?v7(%i`TDMwmk8&T}wT&;?_sS9kJq$cyZUG;?uF>)9RU?c=3g=3)7w%`(s-0X0YJR zZM}VdK4XEuYg?#Li)-SJT2-jkCQJG_(fCk{o}C9Oq_1=a+%Y*Uim6tc995VcLlUn| z4wB;rLD|{?1+PV$GHb1i-Xwudu1)k79=w(=Q+suCLRqw$Lz$`d)@X${lfH?zSd(q3 zM_Z=dl)UOnVTB|l+qfQIv#wb>!x;G$t*fK*XdN0r(>-JhS$A?FyVnL10ogTV_1fLU zC#<@>R>{JkV>Q-Ak!vs0>Zk>t)$v|aEkE`oaNYvrOGXYGc@U`-7wD!nW7P#-$C_3G#C7$RUbhRF>CHDFm;4gOAgzwI zzhX-(@!$6q!pX1U`;N8rJl;HS!ETcQ0}=4>W)Ur9Q21wPU$N0IZPuSU?u(( z^=7T56-vPsb>^=5EOC3WS&5+)y^2HtA0e%#gz53<13g6?zNx`I>bQ zO=PXXyGYfRxKt|5kV0U*URTHwD(pd;cJ&bLW0%)q^v}3^vv!-3%b#V)SO~4jSRs}~ zd$j-9aY+o5!a!f?Nd5{#kRts-gBW1q*{BG8Mo7GVDG(Avm7;HK3_OYgK};Nl+(pC^ zS_d&HwP~U>6W*LPGDc7%(h1>8Ff^F3XxZ|9q$dPgapePuMhh!nIaJ;{KXvgzGm+}`o=k4#c-+%eh?v7h^w}UZzr`pvWv%9C7o)m9<=iHp~c5mF~dAoO(|K9lB z@rBcW`1+DHw&Uo{^84LC_I}`fuBh%- z#ox5uw#}8llQrG=zZMq#G_UkYUeT;)?#!Kr`Kt@XcW=b=c1}0`thj8Je_U8J$G@}p z4~FjzFZ%xE^1aJHE!y~`u=tw?-aepis{i4@p9g;sywCsXNNm%w`;CtZkH-p+|J|0K zIRD0}dIpvQ!;b=?j{+ezJQfdJjd`wu>3(!-Jbr5Yanbs3w%u-;547m$_WYhR+;aLD{Ee zT+zCl*3Vwza>_mvx%HdlS+9I6W43NSf4=T+(L&MPjW@d&%NHBp+q$&%z3N-tcQU40 z)T~#YIP%}Em<_yB6?bf!i>QtYRjByccx(2_O5w+QG7&!DEC^S~UbN^jhC$=5iE*nng?KL( z=ab-)#xV)UD!lxf@|f9cVnVMuhpQJ1@&U4j#flNLwckoE&>(j_*fUE<)ZX9UkJ(I` z*l5-ebT)n{E+b-_J8BZeXog5y(Wn`Pz(_Zv!A?PTu>cM8J$W*3E5J?UhqaKg%ng5eP*cx9l_aS@#@2tITbi=jRl)u;QV0EATYG z*G|24YI=NGD28M+b?SNfB*}7FFvZg46Nx9eGL*({YjSId??JkDNoH2Yl$Ii} z6f@WpAtl{-c_~A(NSTs#kIp!F1<6K4Dys=98=p#sXLj`I2ZX`fcuj(10xnAm|`c z61izDr}RXg6wq3Wy#=`)%Fl#Dnh5zjLoKo#mhbWMD|ihz@Zu}P#d4}`BPl+$O^WB$ z3`6t=b#01!(D;*6AD&XrT#UE%{$F5&_$wxL!2c=7T|(?L>iiWGo+w?@EAVnV)t{k_ z#U;c)Aw|O{4-l*ksjbNQ)sO;2Odawt%F0(L7(tLO1XHi%Q3O;(q2+CgQ$5~Cp5BW%z~z=6 zxqD-7fNLo39$xMk#)xZ7sA*w`dS`VGFx^o--K^3A4Sa@;6xdLiVsnobci zxJ|u;8m+joyfI1~F!|{=Y#O7y0o?G^$&YDhBX2_sqFFU*IG!kIW!qzCz+`m=#{#XR zVM--i}AcoH#QQd{=^p%DgGTyA!5iMOR8CDY-p`_3{*6>%*V& zmk3BRWmB^@$E=&7kD_RK%v!ET#h6vpqt!8M^;F|yn`7$Olj5?uy19{sSC{gaj>U^# znL0KVeXHY1!MZtq?!f$&#k$1-bhJsECWRv=@mzk^iu~;Kd~8JLbn_vz*9^YnBKQu#VXP%&Im-lH#)aHq zZO-)yXj@+TyOi#=AT>X={o%D5{0Pzu($X{2(hKV?#uFgKucZJ;6YG<2roBT@D@r}L z12-AxL+Qn-bk_2cv}aCQ2dbcPS6aVH({kzU^=WNR?@L))da_06Vc+!T}BEmZ&EphMhaR2aWDz%`TT{ zxl4>#>b_R1drFObX`8g&P`^k!q)N$&jh78ZZt5=OtyND-l_uq_>{Z$*Cr2NvX;sJg z--0%{TrPWXut98V>^R|~J>3)7psce;f{_u3-c2c620IDP4cKpmIY4+A1p+d*QE82d zj4Rk9mVHAJJGiYr$nn9UAj)AQvOx?%j<>_k0OG8F#HXMn_W4`vD5N02Z!Cy1*Y?%y ztvO&v9I9^G0dEkuplk$NyrU3yU6+CiOf@FNm&J%2z$R@V;-h^~G=}AigS!X!?cY0u z3`HT+jV++__%7No(l>q=JYJbhH^=Sqcu#dQCPny~EksSm8`~qW#i*~{` zd&LU0vL6v8Pot-`$<^7>c24|0ZNN1G z6Q2<%*ko8(1+Rfv^t81eZ)rN$)YbySB9gLPfg1t;IMD$@lgcm6b{-T#N-%k8d@!*? z%o2rg6xea11=c|w9(QAthYW-+Rz9BwbBpLp6%}G#jcAx_F*p+CK^+zLRaMnuRTZ1YS`=m>{wsP^6Bnwg7_6(RG)}atsz%(=M5SOI zqA=7=&I_@UMqCMu`XG$@#WB!99sbvdc@N8gHY_-prIs;j{apbW=3}E>VI{&S5UX41 zce1bI3&Q|96rR8rW?to@P1*=jpzwfF6#xJI_jUs)`2fl;44Y>BgORmIw@>ObXp+8) z`83Ilg2YMtBqnI`kIEVFK#_irAt~1+1?7H9kCv=K?vOK%rJI{XZy~i)Io1nVlcGs_ z?q^e|KaJ-nxmK&v5b*VP{0fqzTP6+LIjSy6AN7d#VzJ(qWUlsV?Ia+=tnKh-+81k{ z7c#VZwtP7yMIpOWaGIlz`Wi6_7(Y53J=&iJ9>PJGAc;`^hG2_`ab{{3#X$|3r|uH$6Pr-4(0qk%JoWkW$(A*u->H~BHD(!Qrg zG=RdSt#`F*j5I?Y?oZMU7}boS$r|$Rtrz>5q*>Juip0#+n8j2W3pAMt66I%ff+-a+ znXyp!GV$_BSM%xI8nriF`E?SO4y8#iJuwo_IC$5}8PW$*%2vlDIX&aSD%?swL zj^Goc1K|;OKnagv^{UY$5=#YMSL!rC+X0~=Y?M4s{Z*Z0Xmh`Y-SEBCO9wUMJ>smA z!_t4+CkK54BZ0<{Z)5`YjWsM14YV`!!{g&2X5!bcy~`HFej>gwCj+iwQAGt&DgoCX1B|rw=SOn2Ul;!zz|{vH@;wzW2a^Smfl!g3VCNJm?<%hTD}<`iEI1jZOG9KsT<({ILb z5G{Sq8k{K5b4KA<<*{f4u$U2pN8M+r#8@QV0+~23j5Qf6Lj;4tH*t}Kbc{6|05^we z49W~~Efh$f9sR#1tDjth8yfT|yV25fw5@L6URT?$UHkfL?5n`Ei@?;_+}Pz=14PCY z7nN}|SW`q3&(hSIT1gbL>crk|_lPN7(}}Tcn>)iOfv}m(Y6D-ckXtz9oV~THy}J-iy3%&g?z1pj*OSbCN>(R{)rO;YhPukLIj2TbId< zK_*oVGv>nXelx|HO1T9wY}?-ePd&)C{$wI~uvA75ic!D zBXb>7EltZpGfUd{7yP24He6IKy~{%1Q8;-e|&sl|6;vr-?wbtulMfV zb#wXelr38~(kokjHv_xmo5atuSRbpFg_qHiPdT3Y&Szx&+>t#!Jah1^W(E`0GO{d; zvZikNZvFh#KioTCsa75Qi>CYhkB%?x$Ie7E@1_lkOo?_mGCU?KYk7HDsD5HEes?f# z-$0wb`TC~sjOAALO!lnrp|zBDdJBG$T{zqHyFEtd_-SQ!>$Gcb>vx=YoUw|Q9}X=^ z@1Oo*C|=PLFKda}TOJ58d)q^6JDD@A|KI3KdzXcMtoJapX*2y}5U)UE8v=!#)JG6h?^?iA)7kdmwxR5!yT&A4}ME;R3!OWP>s=J`7BS zJqr94fWD)_P*52qdzw*b=n~oC0T9n?zJVaC%}&&H1T-Ns5iLBXYjr@elb_3xw9n}; zL~|wWtKtZjy^wz*x=zBPJUHQE>?x=k(`ZmrvYV+X!^Vqwjh!CXkQ~HZB^L}uNjREQ z?ThAt^(77B5co~ara8May`|Chw7tamU#7Rjs*2~J9}V{tXJTAL0kECGJv)k^m+OUr zdCDTu3#Ml`JP679J+(bin6y(h6vo)P2b09Mc-Q8rsPZGGG@c6jNNxEq&Y1KV& z6rj5cfzD&W(2TU#8jp3)1j8cc2i74^qa3!8XK-V?!7z3hg9(x5;-%=;4pW*Dy8PQH zl_+IiUuiqT2Cwx4i{*f_TIR}3*xR?+8-aqsDTz4$@rTHBT<#($d=2rh;@bhN` zt>4_RnWrsTT<4C0;e!!5Oy8T_tsZRM-9xJUM+lf1e1E@gZ^S+>{|VCHqdcO~pOeEz z&c`_i|71fdCRa8P%;e?4zDt0H&1-Vg z+P3PFvOG2^o8Sf3-hz0J5wCwyyKQJUd_*xHepS2kdT0@3m&XQ3>a$MD&&8%&Ju_`F zAP8~7Yi!iia#CS8pVt2CU()`fwDuRLwZABR!z;a|Ct-^bGNWV(_0-Eu{^4h{HgwkY z6zCn$O9D!zcz1p3U8I!3kY8|%ypxx<(0C)a;!AV4erfKuFU{Rfx%kSC`s}pc z4QjoE`I1Gd1a86tQQy7=rk=JXty$|}?|nz>>B=53n^xeJSBHbwn%)YQE2eU#6>Vg> zwUi6XR|vX9T0?ehaoXJ=w5u=J0P~WQFkPxTz>D13f8ovCfkN%D+t;=fF+wjVN6504 z#jinQ&{K>n>uK}_<5um>1hiaUyM{~2Sy`M+G3I7XYF-(6t7ca%(is-6s6|UzyB3zW zs+OHn-CC4O+EvDRVM%WE%@|#@zP>)+cmmG9mIkNYDL6^YCM>;|AqPAPSZXM1#a#X8 zG*=}ql%29o_7rGyR@!UO6OF=!a?;;Cott#=foVzjzny*704n0cRxaMpC5(YrUJiMK z5XF?tMy)IbD|sjH&4K%msPzEuKe?m%X;#r$-hAN3J{>nqu-MPjAnkR~cNm_v0nc)b z^xdXVw#apwG`v}p2CPME_8aX2Y!B3Dc(ar97*^dRUmCF zzNhu%>~`=o=A`|$Eorr#3#;wJ9bc?_fp0Qf(LcfDut1PQtV`2xKJf^K-Wzg zB{PB*3=n~usgzD3a)eJ~fS}z;H+9rlABpo|&7zmkV_sV$bXG_=DHt{}b(0x-pUMR9 z>>#5kVm*X;Ms>f`jrAJgg160u35R{U7xXsOQfARpI^a`!_!5QM=tA^ge>@*yurq z4vEd>Jgjmjp<#=fYjzENAwX+1iO>B8s)-&vHLgUG#stuwky8N986eppAYyX@Iu0Cl zPz`Z`9J`@*7=Zu!l|V#sHfS}Ns;Uyam@r?zG!ZqqVEHQ^7Uc!1+a&fdw}7bWqR7fR zh3Jn?CJ#9fD^J#8^HbNUKq*dU3)sumr3UCtja}-4;>lg&P{5}IN!dG22QD!63pRR2 z!V{!Yr|nDt5!-SR6s{M?!VoQRC_*@VMZp7B|b#3JN#95KaKvj zrp+^SV3OK)s@3Cy0+G3|vxE+|VM`K|jLqC2oezQZdITp(+emr|py2u>3Z`!gQXIal zD5oS$C+o;j8^V@wys@<{VL7?0?=ox==}sb|>14h9E?_Nx9f8y2G=1SnYm5BXEHzs2 zd~@U7@cC>y&ODt4aPAUt4gx@n$Knxyfz_x!9$lSWym1U zBL?9l1L7r=JtTQRI{9~${XT+pCtbN535iW>lLg&P_kW>xwj=n*Dek)&-wiA@E+~IE zy4W2T_o*-KTPj~V`TooIM`8_Ids}M<}$TqB$`Ig)R)!M(qgZ1N6*S3@tt+R+^l3J9k2NISOb& zudbhuLh08p1)+~7g=-peX!>x=F3ss6!ubwC}yqz13XhT(Vr=4WIs-$>TH)yFh@=crAr)bTQEc4AXHSI3tD zn)aBRQ3pdU#vX{#^&xC~Fia9LkAa)|yZd%RV@+g{^&nxn<{KFgDDp{4q3+1J6tqx4 zbdwP_c?mK2g~918k%74|s>sJE#SGrVS(xD2DI5_+7U_kvl*3<6@WXUcfFFc@6dUS{ zQt1O4zLF5==%R!`lm(9LBf&9Et4^biJlabpaB7s48kWNdBMlAYU!dfl z;ZL~;4(|)iK-*psw{C@c+wPcdovDUD85Ehyt>{ej=49Nlao!wvRD9pO(DcXFA2$7k z@TaZu%EPK~SalqJ?8urvH*;vJnXILo9}8AByL`Uod#CQ6TCTb9F?^(JF2uK9h-F_; z`zB-AWaj=R&rI3dO&(GJpEdQ+?SiSFY~RnCe54y5GzDl#&uX`J(-1wq$q!Lp_AotU z44bB!f0mt3oPBe(dL$9fmZ=&W95DB+NQw!aO`Q!AV;^QU|M%xJ!X^;zk zE?7UxlAr~C;m6@JHh=o=f%%sg+-lLzWozBv=NFTkT>k8~+t#`K+YUTr<=uL9=GD0^ z-#)*<-?=!~xlq4wb+Po`YfE+a-cW0fEDhYxzhC!3(S6JPfd}~y>OL%bU{OmuA7*uZ zmcbR2d}`rr*{sXQem57I`J94Jp_#Wm<@oID{AbX}U+1Tq{}TgE7HQ^BtqCK1vFJ~( z-#xu_T;1AqzaBGOm3kg^U-;mfI&eun9#n%PIPC#&i1NqC*Wd=66JnzZ5?W221fNm9 zi*8*eiEh27KDzar2B~nbHgF!(c}l!ux=J?*XTaemo5tyO&2*h^H%xC3l&_iIq^EJy zQCf|Tnvc<~xQW&zTeBJ6Mkg&C92T2@R_va!O$$F=zbT&G^R1%U#`%4B%NCkHF0Z@k z1C7dj@ACb^KMUXVJt%*0;=?M{d+y^?=WqG$6iu7d>>ljk+{&EETrToVXU44F$HnWW zJvZ&2WpX7F|EU!~{3z>qOgIiaE-L-zsoST1uWiOUEf^R!IJbLl`@AskhJU{oKAAo9 z?KtOQ7rZj}ES0=>c(L|=!C$pJ*z%zJ!)i*H`S^r%5?V)+|yzg#itfX?G zX|V#3-F~m_qmly;@;@p#74x2p<)5EE_N1_6HUhY8IYwxzDNH7`A{q#D;BXR1Jh;N5cTh8~i!?Xz_VQ6nadbUo75yA^4T1 z?AoP_vZVR~ejg?%c`no3hcZ2LRrPrE;7CU-DcoAefZ!lSUVB4=*GlT>kC1S7H@yQz zYKCMzY&y(bG46oUZcC5ZutK0yUUAkJa#cNqpt(4C^VhX5_)H;ib+j|{uHn7P3pLp=NVq$1(1c| zg*BAU#f@H3+~`$lebNg>gijFEx9fGaWIE(cQW^YOHyj|Gn*GeHlghwP(jF=o(O5v3=BDk%UI&8qi?%uj-e+ZSX7l1-i-lrPm+I()(p>DpOPb<4=|gDDsx)OXpHwDi zCSxfDb1D}x=T3O_lP#NeE@r|4qfm(O>~N*%ddjFBqeB3hxmc}Xv0Cd#bCVY-g$nWa z3`%@R*FXopn`bV?*cYeF)xk8Su>iA=)3g;BJ2+0WmaNtX>x!TMj1o+~o#Os*BU9@n z_}WiMx=uTR#0raKXd!SAC>AxS$jR4%9lS|TreGvmJt-d7PA^4-PO^kzu33Kv(I%}t zVy%&MHC+Sg>(T@HH|fK_MZpg#AT8VqvEPh@di*`BMzBI@%(OcmHl0^#cR13h%gEQj z1C`4R#V;aA@WJO5!--960?e={0K38|G%~t9f$Lj+I6NW$cO&2G(-p+|MLblIP;4=) z0MQs2$w=TlBmJwn{ogvN{PnoB_ZfT?@FuW)P;%Fhpk7@=55D z89tfE={B`=qG{lUn073xu2!ZY!pP9T(1_?`fi$wcUs)T5sMOG;rjHIOQdf*d5=XPp z1@bu8PUpcnnQ(k3o{Qn*J)DCXea`XyzaSgTF{fA&OmPeR!uCJ1F$9$V15y$KInK!- z4mi-jXdc@{h?<}h)R$-Ef?dWR!wK4hso;`kz3Yi>Tg#Obq%)~%l}wUZFw9Y~VZro) z+8&IM;3uP|3zHq6P%E&lkO}4b2>g(sa$y*hkL2f-gM=Ud2jv_>w>W7B2bYI?EONvSkp84SQLt(+Ga znsC0cg>n$N`s~2)q;#M6Ol-I`-RC`hf^?v8g1_Yy zK44a51@>F&&`4}>^kLtqo~m2d6s_Z(wy4lJ7Oah0j$seNUz^BlG2E_ngil!A;~`}X zz)o1nm7Xl5a99L04`nA|B&>!lWkMipdO{Mit_sM;j5^c8PLyRf#bj9yy(gT`!m#BG z_=7_e&`ImZW*fUAVxBLA{(haHME6xy^ za9<{KcrdAij@CfDc_7`g4xr` z-Sn8jwEGFu4MhP{&54@IQ42MtqsCgW1zj!!u>S$Yls~|Dea>`_)73X~p>bSSI#>Gk zYjMZfw_lqrU$EQ_KHA<8+uooaJpE|D4dw>Aq~!{e#UkasvP}cw(53v z%(^K(#d$j`X5E;cvh{Z6Oz!UppJkJfpTlJzR1cj&Q>L1qp(n73Us zRn@eRP-4s#&NXW%N6lO2{R_akds`Pu78D#?(f!`;#qCSxrN$-S`^`&x?-!^=%@3_D zWZAmo7yAw^b^q|jvi;}}ZY-96y?!~zHFx?u7w=sBc3(Wl_0ZvZ@5WS(^d= z>s7S?$FeT+_sDp6-%|b3)%W)t6pyGk=!|e}B)$X(ExeMyW-q^W5^=!Y|Kd9yf zmW3g_4Jw!YCR5+M3H8kr7}Pe$HrFj@?*gp0Zu?%#-Ij&zi(8hr?uqAhf3$Va+|{MM z@2|gqId-7yo%OT%vwPoNAKSVome;MiJ*wnU^Ssj;hK7uKfBcSfZo_=@LjFR-y-f?b zi^@{n(z*8!E!8|QKWKa~_~D5M2UO1)wdkzc)Ay)nFxE2&g$k`ZxY4=;dP7gRUCone}nGfeYx)D5;d$VQeW(O2Km9y|RU@#ik{Gsp|dg#W(N!A8h@J^Fy4B1|?%( z%+nt~*t#0`8&r|#}-xC^7+#}T-G`0r%E|T zBW_j>tjFh11?oIK*{|^mgx{B^Yr;e+f=Z99Yq zHKhnYv|AAVP0{AIYT{u=w$|dEH8b~{Q zOUkUt3JFqCEf4h+9eM&@3CEoVXxamZns4hVlugk5Y?Ds6{ws6???1FRh&hB%w|y)#`ddyEj6ZWCxz0 zJ)00W+DUPvofJ3Pxj9`-eLg-CGJ?9WeO$b`t?u5Ck6|wft=7hPa9f8!+1hx>KZ3(m6 z+Y#N?27`X=lEckP9`}>y8S_Xt9NTb+C;8k)#S^k`bkrx~Cru;|D`EI!Vk;BAKT5Q; z6Ba!5BP`mH2XK3W^AcJ1L^m;)PI8J1q2c5Ur4`ZkH=~;+^6Cd4c};h8)z3`O+@_T1 zMs@-k^3g)L^l7=MosJlVBZ9#Kvqs_AKbTcSi*bM&xlFS(aLQMF_yL9C=;kIEVPId= zNJp2*t3Vt8BS!fXHUEs4v)u zk~}*#8s-_p6D(i%Lib$-eS*Zuj%tz4kt<@y=|(C2s1}_6mHZ78IGK}MAQ^jHgCAj~ z@%tstH4kw)_W#R6T%J42lpA(J}l|jvqdbN{!gYrmwD+V3ut( zg&7LyQ65fer_Mu-~;IIl=g%rXZf~pmRq0}7P0fm5gOOI|!_!|or-G^*HXLo?0mpub1!r}T4=#@CXwndrtSjwy z_#?w9b5d-ip5wvru%#q&BV}pYj~`Le>CRSCzCh?@0xfXfkHa)DuaGweyS-}=JqH8V zf|^6mqzKm{oK%$V-n>d*W;_~_&ScaAV^yM|q&{Q6$)dUZCNfxiAXc|%%|QaSCYp_J z(?3urG=q#@b+r{!QqhwP=8lWn^=Fig{>XlMg+r$f^p(HdqI$}Usw%l zh9%nXp``ul75zGvURghE!f22*D=2?>WPFr}Er>c<_u{ycFr4Q{CUk72!!$<>T!s&M zR{GB6 zn#_-j4yz}Ik#?2uh9F_ecALQeTh`fKpK|MLHB*hxc5*p|Xhc@-%yDW9bNQUz%VfKz z`HZ{Px#&XSLifUJ_j=XpLrd^&yuafE`_klt`Uh9l?w$`{e-KtL_N%@D)qgqWyQ21w zs3Q|p9>jZJNed0HMe}eV*W~O z%PzINo{4$anIx5|c1bFZ-|XsBJ0aV_zTk>FqQovm)ZuY;oNQTg8ku+HG1FtMCQ52& zE~y>n)&7k~)R#}iZLdC%)SiKu;D00pV?t299C~zFiCrcmnfT>v>cs1@%damBZ{W9m zX3aMbs6r7_@~mL?i7jW${%;{nXZBMrdu~!zthx*qQQ{lkg>%4N>`v*7JVtSi@|qB! zyNN;J^D9a-Gj`^X#(tQB@*b~a)GewHZtEtJn>u5IX@*96xF^QzjcELVI%`$);cE-ATts_nWOwHSA5Tk1qa zqdhpMm`v8#W30iCJ_;3Jl~)z{%0MYklEy<}qa@+$APpCh3!hugL^yd5Qv3ENUy= z@6jX85^YO|8EDSpkxm-G!2$}idt|?x#FinaL#CYr*-J%Cp&|SQwQ~hb%9t!t)_}^B={sld zoSpaG=~-xvZ`$+3P+-oyiY362rRJ5#9pz85aHjmbm){wg-xn)+d7&{@QWGzzUC!DG z3#PpMPhstss}@wpv#Xz!Zu$Kz=rb$*WL0-<<1+^H;Vxh^Mb>8gN zs`RO)b@7s2iw9yQhvNlDma|@YoSXN=R(NwG&f67=X5Gs|=@Xk{rQ7?7xD99FKFPFw z?fP5SzxKvkZ_Exn%3L4IT>oUl_TN8D=M4z6&Gf@l?%C40-OE<-xf+_|wwASPSXche zKwj8!_qD~kyKlrR_Nx0&KB)Lu=+Jh5 zt$lq2^}ar~W9r7XCK@y2%B9q$b6^yN_p+}mfay@^wR=k4$oKk}DQ!~%= zp9$H#@N6T;XZ?(`vH$kJ=c<3MkJ_7Wy*Vp?GkQDv%{Okpfy0yQ%#Zb#Cf}NzZJH~IXNo8%)J*YDEj+*Lsnx>2#(!GG;UEkF zc7IdWrxt-PdYWVAcm6VKJ-_wo{tQ0%X`X}6dAji+U-0w|p0ge`p=?>LdZMrrpLuQma^4uKOV2c+7`FdVA?)2^Et9<%Kp^K^UeHIYZbro>3M!5U-9%v r6<>^K3E# rendered from data/interpreter-scores.json: + weighted totals, hard gates, two recommendations, sensitivity (never typed by hand); + (3) REFERENCE-TEST-CORPUS.md - region from data/tests.json (Language tests by folder and file); + (4) 09-synthesis/new-language-design-canvas.md - skeleton rows with lesson counts from data/lessons.json. +Inputs : data/members-tierA.json, data/parser-methods.json, data/interpreter-scores.json, data/tests.json, + data/lessons.json, the source files (read-only, for call extraction). +Outputs : docs-out/02-language/CALL-GRAPHS.md and regions; docs-out/09-synthesis/new-language-design-canvas.md +Run : python3 docs-out/tools/gen_coord.py [--count callgraph_nodes|test_files|test_its|options] +Label : APPROX (call edges are matches of a member name followed by '(' inside another member's line range). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys +from collections import Counter, defaultdict + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +S = "src/System.Management.Automation/engine/" + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def fill(path, tag, rows): + t = open(path, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()" % (tag, tag), re.S) + if not pat.search(t): + print("region missing", tag, path) + return + open(path, "w", encoding="utf-8").write(pat.sub(lambda m: m.group(1) + "\n" + "\n".join(rows) + "\n" + m.group(2), t)) + + +# ------------------------------------------------------------------ call graphs +_src = {} +_members = None + + +def members(): + global _members + if _members is None: + _members = {(m["file"], m["full_name"]): m["members"] for m in J("members-tierA.json")} + return _members + + +def body(file, start, end): + if file not in _src: + _src[file] = open(os.path.join(ROOT, file), encoding="utf-8-sig").read().split("\n") + return "\n".join(_src[file][start - 1:end]) + + +def graph(file, type_suffix, root, depth, limit, include=None): + """BFS over members of one type: edges m -> n when n( appears in m's body (n a member method name).""" + key = [k for k in members() if k[0] == file and k[1].endswith(type_suffix)][0] + ms = [m for m in members()[key] if m.get("kind") == "method" and m.get("start")] + names = defaultdict(list) + for m in ms: + names[m["name"]].append(m) + edges, seen, frontier = [], {root}, [root] + for _ in range(depth): + nxt = [] + for n in frontier: + if n not in names: + continue + b = "\n".join(body(file, m["start"], m["end"]) for m in names[n]) + for c in names: + if c == n or (include and not include(c)): + continue + if re.search(r"(? {b}") + out.append("```") + return out + + +def callgraphs(): + P = S + "parser/Parser.cs" + T = S + "parser/tokenizer.cs" + C = S + "parser/Compiler.cs" + R = S + "runtime/CompiledScriptBlock.cs" + gs = {} + gs["tokenize"] = graph(T, ".Tokenizer", "NextToken", 3, 40) + gs["parse"] = graph(P, ".Parser", "ParseTask", 3, 45, include=lambda c: c.endswith("Rule") or c in ("ParseTask", "ScriptBlockRule", "SetTokenizerMode", "SyncOnError")) + gs["rules"] = graph(P, ".Parser", "ScriptBlockRule", 12, 200, include=lambda c: c.endswith("Rule")) + gs["compile"] = graph(C, ".Compiler", "Compile", 3, 45, include=lambda c: not c.startswith("Visit") and not c.startswith("New") and c not in ("Compile",) or c == "Compile") + gs["compile_body"] = graph(C, ".Compiler", "VisitScriptBlock", 4, 40, include=lambda c: not c.startswith("Visit") and c not in ("Compile", "NewTemp", "Constant", "GetLocal")) + gs["invoke"] = graph(R, ".ScriptBlock", "InvokeWithPipeImpl", 2, 30) + return gs + + +def write_callgraphs(gs): + L = [] + w = L.append + w("""--- +id: LANG-CALLG-001 +title: Call graphs +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/members-tierA.json] +--- + +# Call graphs + +> STATIC-ONLY — behavior not run-verified. Every graph is `GENERATED` by `docs-out/tools/gen_coord.py` from `data/members-tierA.json` and the member line ranges (`APPROX`: an edge A → B means the name `B` followed by `(` or `<` appears in the body of `A`, so overloads and same-named members of other types can create false edges, and calls through delegates or interfaces are missing). Use them as maps, then read the method. Sequence-level narrative: [LIFECYCLE-script-execution.md](LIFECYCLE-script-execution.md); stage data contracts: [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md). +""") + w("## 1. Purpose and scope [SPEC]\n") + w("Entry-point call graphs for **tokenize**, **parse**, **check**, **compile** and **invoke**. 'Check' is not generated (the semantic passes are visitors, entered through `PerformPostParseChecks`) and is given as a hand-written chain in §5.\n") + w("## 2. Source map\n") + w("| graph | entry | file |\n|---|---|---|") + w(f"| tokenize | `Tokenizer.NextToken` | `{S}parser/tokenizer.cs` |") + w(f"| parse | `Parser.ParseTask` → `ScriptBlockRule` | `{S}parser/Parser.cs` |") + w(f"| compile | `Compiler.Compile` | `{S}parser/Compiler.cs` |") + w(f"| invoke | `ScriptBlock.InvokeWithPipeImpl` | `{S}runtime/CompiledScriptBlock.cs` |\n") + def sect(title, key, note): + e, n = gs[key] + w(f"## {title}\n") + w(f"{note} Nodes: {len(n)}, edges: {len(e)}.\n") + L.extend(mermaid(e)) + w("") + + sect("3. Tokenize", "tokenize", "`NextToken` and its scanners, depth 3 (`Tokenizer`, name-based).") + sect("4. Parse: entry chain", "parse", "From `ParseTask` through the first rules, depth 3.") + w("## 5. Check (hand-written chain)\n") + w("""`Parser.ParseTask` → `ScriptBlockAst.PerformPostParseChecks` → `SymbolResolver.ResolveSymbols` (visitor over the AST) → `SemanticChecks.CheckAst` (visitor; for class methods `VariableAnalysis.AnalyzeMemberFunction`; for `data` statements `RestrictedLanguageChecker`). Compile-time: `Compiler.Compile` → `VariableAnalysis.Analyze` and, for the root block, `Compiler.GenerateTypesAndUsings` → `TypeDefiner.DefineTypes`. Evidence: [02.4-semantic-analysis.md](02.4-semantic-analysis.md) §6. +""") + sect("6. Compile", "compile", "From `Compiler.Compile` through helpers (the 63 `Visit*` methods are omitted from the graph; see [COMPILATION-MAP.md](COMPILATION-MAP.md)), depth 3.") + sect("6b. Compile: script block body", "compile_body", "From `Compiler.VisitScriptBlock` (called through `body.Accept(this)` inside `Compile`) through the lambda builders, depth 4.") + sect("7. Invoke", "invoke", "From `ScriptBlock.InvokeWithPipeImpl`, depth 2.") + e, n = gs["rules"] + w("## 8. Parser rule-to-rule edges (complete list, `*Rule` methods reachable from `ScriptBlockRule`)\n") + w(f"{len(n)} rules, {len(e)} edges (`GENERATED`, `APPROX`). Each row lists the callee rules of one rule.\n") + w("| Rule | Calls (rules) |\n|---|---|") + adj = defaultdict(list) + for a, b in e: + adj[a].append(b) + for a in sorted(adj): + w(f"| `{a}` | " + ", ".join(f"`{b}`" for b in sorted(adj[a])) + " |") + w("\n## 9. Guidance for MicroteamShell [LESSON]\n") + w("""- **LSN-CALLG-01** [REPLICATE] (area: NLD-TOOL) Keep the five entry points (tokenize, parse, check, compile, invoke) as separate callable stages with narrow signatures so each can be tested alone. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`. +- **LSN-CALLG-02** [ADAPT] (area: NLD-TOOL) Generate such graphs in CI from the shell's own sources to catch accidental coupling between stages. Why: the PowerShell graphs show the compiler calling parser-time helpers and the AST calling the compiler. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::VariableExpressionAst@L9404-L9624`. +""") + w("Verification: graphs are produced by the script from member ranges; `ParseTask`, `ScriptBlockRule`, `NextToken`, `Compiler.Compile` and `InvokeWithPipeImpl` exist in `data/members-tierA.json`; spot-checked edges: `ParseTask → ScriptBlockRule`, `NextToken → ScanVariable`, `Compile → CompileTree`.\n") + open(os.path.join(OUT, "02-language", "CALL-GRAPHS.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + return len(n) + + +# ------------------------------------------------------------------ scores +CRIT = ["fidelity", "sandbox", "startup", "throughput", "debug", "aot", "determinism", "memory", "observability", "complexity"] +CRIT_NAMES = {"fidelity": "Behavioral fidelity", "sandbox": "Sandboxability", "startup": "Startup latency", "throughput": "Steady-state throughput", "debug": "Debuggability", + "aot": "AOT-friendliness", "determinism": "Determinism", "memory": "Memory footprint", "observability": "Observability", "complexity": "Implementation simplicity"} + + +def total(opt, weights): + return sum(opt["scores"][c] * weights[c] for c in CRIT) / 100.0 + + +def scores_rows(): + d = J("interpreter-scores.json") + wts = d["weights"] + assert sum(wts.values()) == 100 and set(wts) == set(CRIT) + opts = d["options"] + rows = [] + rows += ["| Option | Name | " + " | ".join(CRIT_NAMES[c] + f" ({wts[c]})" for c in CRIT) + " | Weighted total (max 5) |", "|---|---|" + "---:|" * (len(CRIT) + 1)] + for o in opts: + rows.append(f"| {o['id']} | {o['name']} | " + " | ".join(str(o["scores"][c]) for c in CRIT) + f" | **{total(o, wts):.2f}** |") + rows += ["", "Justifications (one sentence per score, label in brackets):", ""] + for o in opts: + rows.append(f"**{o['id']} {o['name']}**") + for c in CRIT: + j = o["why"][c] + rows.append(f"- {CRIT_NAMES[c]} = {o['scores'][c]} [{j[0]}] {j[1]}") + rows.append("") + rows += ["Hard gates (H1 reproduces `CORE` observable behavior; H2 feasible under the deployment constraint; H3 policy hooks possible at every boundary of [STAGE-CONTRACTS.md](STAGE-CONTRACTS.md)):", "", + "| Option | H1 | H2 JIT-allowed | H2 NativeAOT-required | H3 | Gate note |", "|---|---|---|---|---|---|"] + for o in opts: + g = o["gates"] + rows.append(f"| {o['id']} | {'pass' if g['H1'] else 'FAIL'} | {'pass' if g['H2_jit'] else 'FAIL'} | {'pass' if g['H2_aot'] else 'FAIL'} | {'pass' if g['H3'] else 'FAIL'} | {g['note']} |") + recs = {} + for mode, key in (("JIT-allowed", "H2_jit"), ("NativeAOT-required", "H2_aot")): + ok = [o for o in opts if o["gates"]["H1"] and o["gates"][key] and o["gates"]["H3"]] + ok.sort(key=lambda o: (-round(total(o, wts), 6), -o["scores"]["complexity"])) + recs[mode] = ok + rows += ["", "Recommendations (computed; label `INFERENCE`):", ""] + for mode, ok in recs.items(): + rows.append(f"- **{mode}**: {ok[0]['id']} {ok[0]['name']} ({total(ok[0], wts):.2f}); runner-up {ok[1]['id']} {ok[1]['name']} ({total(ok[1], wts):.2f}); candidates passing all gates: {', '.join(o['id'] for o in ok)}.") + # sensitivity: raise one weight (others scaled), find smallest weight where winner changes + rows += ["", "Sensitivity (weight of one criterion raised, other weights scaled to keep the sum at 100; smallest weight at which the JIT-allowed winner changes):", ""] + base = recs["JIT-allowed"][0]["id"] + cand = recs["JIT-allowed"] + for c in CRIT: + flip = None + for w in range(wts[c], 71): + sc = {k: (wts[k] * (100 - w) / (100 - wts[c]) if k != c else w) for k in CRIT} + best = max(cand, key=lambda o: (round(sum(o["scores"][k] * sc[k] for k in CRIT) / 100, 6), o["scores"]["complexity"])) + if best["id"] != base: + flip = (w, best["id"]) + break + rows.append(f"- {CRIT_NAMES[c]} (weight {wts[c]}): " + (f"winner becomes {flip[1]} at weight {flip[0]}" if flip else "no change up to weight 70")) + return rows, recs, opts + + +def tests_rows(): + t = J("tests.json") + lang = [x for x in t["pester"] if x["path"].startswith("test/powershell/Language/")] + folder = defaultdict(lambda: [0, 0, 0, 0]) + for x in lang: + f = "/".join(x["path"].split("/")[:4]) + a = folder[f] + a[0] += 1 + a[1] += x["describe"] + a[2] += x["context"] + a[3] += x["it"] + rows = ["| Folder | Files | Describe | Context | It |", "|---|---:|---:|---:|---:|"] + for f, a in sorted(folder.items()): + rows.append(f"| `{f}` | {a[0]} | {a[1]} | {a[2]} | {a[3]} |") + tot = [sum(a[i] for a in folder.values()) for i in range(4)] + rows.append(f"| **total Language** | **{tot[0]}** | **{tot[1]}** | **{tot[2]}** | **{tot[3]}** |") + rows += ["", "Files (path, Describe/Context/It counts, first Describe names; `APPROX` regex extraction):", "", "| File | D | C | It | First Describe names |", "|---|---:|---:|---:|---|"] + for x in sorted(lang, key=lambda z: z["path"]): + names = [n["name"] for n in x.get("tree", []) if n["kind"] == "Describe"][:2] + rows.append(f"| `{x['path']}` | {x['describe']} | {x['context']} | {x['it']} | " + "; ".join(n.replace("|", "\\|")[:60] for n in names) + " |") + other = [x for x in t["pester"] if x["path"].startswith("test/powershell/engine/") and any(k in x["path"] for k in ("Api/", "Basic/", "ExecutionContext", "Runspace", "Pipeline", "ScriptBlock", "Variable", "Type", "Debugger"))] + rows += ["", "Engine tests that also specify language behavior (selected by path keywords, `APPROX`):", "", "| File | It |", "|---|---:|"] + for x in sorted(other, key=lambda z: z["path"])[:40]: + rows.append(f"| `{x['path']}` | {x['it']} |") + xs = [x for x in t["xunit"] if any(k in x["path"] for k in ("Ast", "Binder", "Parser", "Language", "Tokenizer"))] + rows += ["", "xUnit tests on the language layer:", "", "| File | Facts | Theories | Classes |", "|---|---:|---:|---|"] + for x in xs: + rows.append(f"| `{x['path']}` | {x['facts']} | {x['theories']} | {', '.join(x['classes'])} |") + return rows, len(lang), tot[3] + + +def canvas(): + lessons = J("lessons.json") + cnt = defaultdict(Counter) + for l in lessons: + cnt[l["area"].replace("NLD-", "")][l["verdict"]] += 1 + areas = ["LEX", "GRM", "OP", "TYP", "SCP", "ERR", "PIP", "MOD", "CMD", "SEC", "TOOL", "RUN"] + names = {"LEX": "Lexical structure and tokenizer modes", "GRM": "Grammar, AST and semantic checks", "OP": "Operators and precedence", "TYP": "Types, conversions and values", + "SCP": "Scopes, variables and sessions", "ERR": "Errors, streams and recovery", "PIP": "Pipeline and output", "MOD": "Modules and packaging", "CMD": "Commands and parameter binding", + "SEC": "Security, sandboxing and language modes", "TOOL": "Tooling, extension and testing", "RUN": "Execution strategy and runtime"} + L = ["""--- +id: SYN-CANVAS-001 +title: New-language design canvas (skeleton) +kind: catalog +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [] +mirror_cards: [] +generated_from: [data/lessons.json] +--- + +# New-language design canvas (skeleton) + +> STATIC-ONLY. Skeleton created in Phase 2 so that forward links resolve; Phase 7 fills the empty cells. Lesson counts per area and verdict are `GENERATED` by `docs-out/tools/gen_coord.py` from `data/lessons.json` (gate 6d compares them with the verdict counts of [SEMANTIC-QUIRKS](../02-language/SEMANTIC-QUIRKS.md) and the guidance sections). + +## 1. Canvas rows + +| Row | Area | REPLICATE | ADAPT | REDESIGN | AVOID | Baseline decision | Approved deviation | Status | +|---|---|---:|---:|---:|---:|---|---|---|"""] + for a in areas: + c = cnt[a] + L.append(f"| NLD-{a} | {names[a]} | {c['REPLICATE']} | {c['ADAPT']} | {c['REDESIGN']} | {c['AVOID']} | | | open |") + tot = Counter() + for a in areas: + tot.update(cnt[a]) + L.append(f"| **total** | | **{tot['REPLICATE']}** | **{tot['ADAPT']}** | **{tot['REDESIGN']}** | **{tot['AVOID']}** | | | |") + L.append("\nVerification: counts rendered from `data/lessons.json` (extracted from the guidance sections by `tools/coordinator.py lessons`); the other columns are intentionally empty until Phase 7.\n") + open(os.path.join(OUT, "09-synthesis", "new-language-design-canvas.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + return dict(tot) + + +def quirk_counts(): + p = os.path.join(OUT, "02-language", "SEMANTIC-QUIRKS.md") + c = Counter() + if os.path.exists(p): + for ln in open(p, encoding="utf-8"): + m = re.match(r"\| QRK-\d+ \|.*\| (REPLICATE|ADAPT|REDESIGN|AVOID) \| ([^|]*)\|", ln) + if m: + c[m.group(1)] += 1 + c["total"] += 1 + return c + + +def main(): + if len(sys.argv) > 2 and sys.argv[1] == "--count": + k = sys.argv[2] + if k.startswith("quirks_"): + q = quirk_counts() + print(q["total"] if k == "quirks_total" else q[k.split("_")[1].upper()]) + return 0 + if k == "callgraph_nodes": + print(len(callgraphs()["rules"][1])) + elif k in ("test_files", "test_its"): + _, nf, ni = tests_rows() + print(nf if k == "test_files" else ni) + elif k == "options": + print(len(J("interpreter-scores.json")["options"])) + return 0 + gs = callgraphs() + write_callgraphs(gs) + sp = os.path.join(OUT, "02-language", "INTERPRETER-DESIGN-SPACE.md") + if os.path.exists(sp): + rows, recs, opts = scores_rows() + fill(sp, "scores", rows) + tp = os.path.join(OUT, "02-language", "REFERENCE-TEST-CORPUS.md") + if os.path.exists(tp): + rows, nf, ni = tests_rows() + fill(tp, "tests", rows) + tot = canvas() + print("call graph rule nodes", len(gs["rules"][1]), "canvas lessons", tot) + + +if __name__ == "__main__": + sys.exit(main()) From e013584a2e7e024a218464ced460969a3e5ade89 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:19:54 +0000 Subject: [PATCH 2/3] Phase 3 runtime documents: pipeline, discovery, binding, cmdlets, providers, ETS, formatting, help, modules, errors, jobs, debugger, eventing - 03-runtime: 13 subsystem documents (T31-T39), 19 sections each, tier B at contract depth - CF-LCY-001 (closed), CF-DSC-001, CF-BND-001, CF-ETS-001, CF-MOD-001, CF-ERR-001; 18 new DEV proposals - tools/gen_T3.py + annotations/T3-docs.json render test and security-seed tables - claims T3.jsonl (43), ledger T31_T39.csv, DECISIONS D-021..D-024, CON-014, VB-024..VB-057 - RUNTIME-OPERATIONS links CF-ERR-001 owner; README and PROGRESS updated Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01HnR7RpZuh9mvNMRTtzxTvk --- docs-out/.work/cf/T31.json | 43 + docs-out/.work/cf/T32.json | 42 + docs-out/.work/cf/T33.json | 43 + docs-out/.work/cf/T34.json | 23 + docs-out/.work/cf/T35.json | 42 + docs-out/.work/cf/T36.json | 41 + docs-out/.work/cf/T37.json | 43 + docs-out/.work/cf/T38.json | 31 + docs-out/.work/cf/T39.json | 59 ++ docs-out/.work/claims/T3.jsonl | 43 + docs-out/.work/files-ledger.csv | 112 +-- docs-out/.work/ledger-proposals/T31_T39.csv | 57 ++ docs-out/02-language/RUNTIME-OPERATIONS.md | 2 +- docs-out/03-runtime/cmdlet-infrastructure.md | 174 ++++ .../command-discovery-and-invocation.md | 207 ++++ docs-out/03-runtime/debugger.md | 185 ++++ docs-out/03-runtime/errors-and-diagnostics.md | 193 ++++ docs-out/03-runtime/eventing.md | 162 ++++ docs-out/03-runtime/extended-type-system.md | 180 ++++ docs-out/03-runtime/formatting-and-output.md | 203 ++++ docs-out/03-runtime/help-system.md | 175 ++++ docs-out/03-runtime/jobs-and-concurrency.md | 188 ++++ docs-out/03-runtime/modules.md | 198 ++++ docs-out/03-runtime/parameter-binding.md | 191 ++++ docs-out/03-runtime/provider-model.md | 198 ++++ docs-out/03-runtime/runspace-and-pipeline.md | 268 ++++++ docs-out/09-synthesis/deviation-register.md | 23 + docs-out/CONTRADICTIONS.md | 1 + docs-out/DECISIONS.md | 4 + docs-out/PROGRESS.md | 16 +- docs-out/README.md | 6 + docs-out/VERIFICATION-BACKLOG.md | 34 + docs-out/data/canonical-facts.json | 89 +- docs-out/data/claims/T3.jsonl | 43 + docs-out/data/lessons.json | 902 ++++++++++++++++++ docs-out/data/subject-keys.json | 43 + .../tools/__pycache__/consist.cpython-313.pyc | Bin 0 -> 30799 bytes docs-out/tools/annotations/T3-docs.json | 268 ++++++ docs-out/tools/gen_T3.py | 117 +++ 39 files changed, 4565 insertions(+), 84 deletions(-) create mode 100644 docs-out/.work/cf/T31.json create mode 100644 docs-out/.work/cf/T32.json create mode 100644 docs-out/.work/cf/T33.json create mode 100644 docs-out/.work/cf/T34.json create mode 100644 docs-out/.work/cf/T35.json create mode 100644 docs-out/.work/cf/T36.json create mode 100644 docs-out/.work/cf/T37.json create mode 100644 docs-out/.work/cf/T38.json create mode 100644 docs-out/.work/cf/T39.json create mode 100644 docs-out/.work/claims/T3.jsonl create mode 100644 docs-out/.work/ledger-proposals/T31_T39.csv create mode 100644 docs-out/03-runtime/cmdlet-infrastructure.md create mode 100644 docs-out/03-runtime/command-discovery-and-invocation.md create mode 100644 docs-out/03-runtime/debugger.md create mode 100644 docs-out/03-runtime/errors-and-diagnostics.md create mode 100644 docs-out/03-runtime/eventing.md create mode 100644 docs-out/03-runtime/extended-type-system.md create mode 100644 docs-out/03-runtime/formatting-and-output.md create mode 100644 docs-out/03-runtime/help-system.md create mode 100644 docs-out/03-runtime/jobs-and-concurrency.md create mode 100644 docs-out/03-runtime/modules.md create mode 100644 docs-out/03-runtime/parameter-binding.md create mode 100644 docs-out/03-runtime/provider-model.md create mode 100644 docs-out/03-runtime/runspace-and-pipeline.md create mode 100644 docs-out/data/claims/T3.jsonl create mode 100644 docs-out/tools/__pycache__/consist.cpython-313.pyc create mode 100644 docs-out/tools/annotations/T3-docs.json create mode 100644 docs-out/tools/gen_T3.py diff --git a/docs-out/.work/cf/T31.json b/docs-out/.work/cf/T31.json new file mode 100644 index 0000000..e127ca3 --- /dev/null +++ b/docs-out/.work/cf/T31.json @@ -0,0 +1,43 @@ +{ + "cfs": [ + { + "id": "CF-LCY-001", + "subject": "command and script lifecycle (begin/process/end/clean, cmdlet lifecycle)", + "statement": "**Command lifecycle (CF-LCY-001).** For a pipeline of commands C1\u2026Cn the engine (1) builds all command processors and wires pipes; (2) *prepares* every command in order (parameter binding from the command line, with default parameter values applied) and sets up common-parameter variables; (3) runs the *begin* phase of every command in order C1\u2026Cn (`BeginProcessing`, or the `begin` block); (4) runs the *process* phase *depth-first and synchronously*: C1 reads its input (or runs once with no input if none is expected), and each object it writes into its output pipe causes C2 to run its process phase over everything queued (the default buffer threshold is zero), recursively; every input object is bound before each call (pipeline parameter binding, then mandatory-parameter check); an object that cannot be bound or misses a mandatory parameter produces a non-terminating error and is skipped; (5) when C1's input is exhausted the *end* phase runs for C1\u2026Cn in order \u2014 each command first drains any input still queued through its process phase, then runs `EndProcessing` (or the `end` block); (6) the *clean* phase (`clean` blocks, only for commands that have one) runs for every command in `finally`, with stopping suspended, and its errors are written to the command's error output; (7) all command processors are disposed. The first terminating error is recorded; later failures are logged as health events and do not replace it. A stop request cancels the pipeline token and calls `StopProcessing` on every command.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147", + "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.DoCompleteCore@L614-L709", + "src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466", + "src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Read@L452-L581", + "src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.Complete@L591-L609" + ], + "owner": "03-runtime/runspace-and-pipeline.md", + "status": "closed" + } + ], + "deviations": [ + { + "id": "DEV-PIPE-110", + "baseline_behavior": "Host-facing object streams are effectively unbounded (capacity Int32.MaxValue) and a pipeline has no time or memory budget.", + "proposed_behavior": "Bounded, cancellable host streams by default plus an optional per-pipeline time/memory budget.", + "benefit": "Predictable resource use in enterprise hosts.", + "compatibility_cost": "Behavior differs for producers that exceed the bound; hosts must handle blocking or rejection.", + "affected_cfs": [ + "CF-LCY-001" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-PIPE-111", + "baseline_behavior": "An internal empty \"pulse\" pipeline is created to run pending engine events when the runspace is idle.", + "proposed_behavior": "Schedule event processing on the runspace loop without creating a pipeline.", + "benefit": "Removes a special pipeline and its state-machine corner cases.", + "compatibility_cost": "Code that observed the pulse pipeline (id 0, history) would not see it.", + "affected_cfs": [ + "CF-LCY-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T32.json b/docs-out/.work/cf/T32.json new file mode 100644 index 0000000..cffadb9 --- /dev/null +++ b/docs-out/.work/cf/T32.json @@ -0,0 +1,42 @@ +{ + "cfs": [ + { + "id": "CF-DSC-001", + "subject": "command resolution order", + "statement": "**Command resolution order (CF-DSC-001).** For one name the searcher tries, in this order and returning the first match that is allowed: (1) **alias** (a match whose visibility does not allow the caller's origin is skipped and the search continues with functions); (2) **function / filter / configuration**; (3) **cmdlet** (a module-qualified name `Module\\Cmdlet` matches only cmdlets of that module; with several cmdlets of the same name in different modules, the first in table order is returned unless all scopes are searched); (4) if application or external-script types are allowed: a name that is a *qualified provider path* is resolved as a path; then an absolute path to an existing file; then the file system search of each `PATH` directory in order (for a name with a directory part: only that directory); then a PowerShell-relative path (name starting with `.`, `~` or `\\`). Within one directory the candidate file names are, in order: the name itself first if it already has an extension; `name.ps1` when external scripts are allowed (also `.psm1` and `.psd1` in non-discovery searches); on Windows every `PATHEXT` extension when applications are allowed; then the bare name if not yet added. A `.ps1` file becomes an external script; any other existing file becomes an application. If the normal search fails and the name contains neither `-` nor `\\`, the lookup is repeated once with the name prefixed with `Get-`. If still unresolved, module-qualified names trigger module auto-loading, and (with `$PSModuleAutoLoadingPreference` = `All`, the default) other names trigger module auto-discovery against the module analysis cache; then the *command-not-found* handler runs; then a `CommandNotFoundException` is raised. In a session whose origin is `Runspace`, application and external-script types are removed from the search when the session state has no allowed applications or scripts, and a name with a directory part must match an allow-list entry (prefix match, or the list is exactly `*`).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239", + "src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.LookupCommandInfo@L656-L793", + "src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.TryNormalSearch@L886-L949", + "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.ConstructSearchPatternsFromName@L1322-L1363" + ], + "owner": "03-runtime/command-discovery-and-invocation.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-DISC-120", + "baseline_behavior": "Unknown command names trigger module auto-discovery by analyzing every module file on PSModulePath.", + "proposed_behavior": "Resolve through an explicit signed command index refreshed on module install/update; keep module-qualified auto-load.", + "benefit": "Faster misses, no file scans on typos, auditable resolution.", + "compatibility_cost": "Modules not indexed are not auto-loaded until indexed.", + "affected_cfs": [ + "CF-DSC-001" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-DISC-121", + "baseline_behavior": "Native argument passing has three styles (Legacy, Standard, Windows) with a name-based list of legacy programs.", + "proposed_behavior": "List-based passing only, with a per-command opt-in for raw command lines.", + "benefit": "Removes the quoting ambiguity and the legacy program list.", + "compatibility_cost": "Scripts relying on Legacy re-quoting need the opt-in.", + "affected_cfs": [ + "CF-DSC-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T33.json b/docs-out/.work/cf/T33.json new file mode 100644 index 0000000..236fbd2 --- /dev/null +++ b/docs-out/.work/cf/T33.json @@ -0,0 +1,43 @@ +{ + "cfs": [ + { + "id": "CF-BND-001", + "subject": "parameter-binding order and parameter-set resolution", + "statement": "**Parameter binding order and parameter-set resolution (CF-BND-001).** Command-line binding runs these steps in order: (1) arguments that came from hashtable splatting are moved after all other arguments; (2) a parameter name with no value is joined with the next argument as its value (a switch parameter takes `$true` instead; a name followed by another recognizable parameter name, or a name at the end, is a `MissingArgument` error); (3) **named binding**: every argument with a name is matched (exact name or alias first, then unambiguous prefix); a parameter already bound is an error (`ParameterAlreadyBound`) except that a splatted entry whose name was already bound explicitly is skipped; the `--%` verbatim marker is consumed here; (4) **positional binding**: unbound parameters that have a position in the still-valid sets are grouped by position number in ascending order; arguments without a name are taken in order; each is offered to the parameters of that position in four attempts \u2014 default set without type coercion, all valid sets without coercion, default set with coercion, all valid sets with coercion \u2014 and an argument no parameter accepts stays unbound; (5) values from `$PSDefaultParameterValues` are bound to still-unbound parameters, after positional binding and again after dynamic parameters; (6) `ValidateParameterSets`; (7) dynamic parameters are fetched once from the command (`GetDynamicParameters`), merged into the parameter table and set bitmasks, and named and positional binding are repeated for the leftover arguments; (8) leftover arguments are bound to the single `ValueFromRemainingArguments` parameter (two such parameters in the current sets is an error); (9) any argument still unbound is an error (`NamedParameterNotFound` for a named one, `PositionalParameterNotFound` otherwise); (10) the final parameter-set check and the mandatory-parameter step (\u00a77); (11) script parameters that are still unbound get their declared default values; (12) the set mask is saved for pipeline binding. **Parameter-set resolution:** each bound parameter narrows the current set mask to the sets it belongs to. After binding, with *one* valid set that set is chosen. With *several* valid sets: if the mask is \"all sets\" and the command has no default set, the call is accepted as set-less; else if the command defines a default set and it is among the valid sets, the command's parameter-set name is set to the default set, and the mask is narrowed to the default set only in validation calls that ask for it (the final check after command-line binding does; the checks made before dynamic parameters and during pipeline binding may not); else, if pipeline input is still expected and some valid set has an unbound parameter that takes pipeline input, the decision is postponed to pipeline binding; else the mask is reduced to sets with no unbound mandatory parameter; if exactly one set remains it is chosen, otherwise `AmbiguousParameterSet`. With *zero* valid sets (conflicting parameters) the result is `AmbiguousParameterSet` as well. **Pipeline-object binding** for each input object restores the saved mask and the original values of parameters bound by the previous object, then makes four passes over the unbound parameters that accept pipeline input \u2014 (a) by value without coercion, (b) by property name without coercion, (c) by value with coercion, (d) by property name with coercion \u2014 trying the default set before the other valid sets in each pass; a successful bind narrows the set mask again.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindCommandLineParametersNoValidation@L202-L277", + "src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController.BindPositionalParameters@L699-L871", + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.ValidateParameterSets@L2532-L2640", + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindPipelineParametersPrivate@L3255-L3347", + "src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L329-L667" + ], + "owner": "03-runtime/parameter-binding.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-BIND-130", + "baseline_behavior": "Execution-detail logging records bound values (first 256 characters) for every parameter.", + "proposed_behavior": "Redact bound values by parameter type and attribute (secure types, sensitive marker).", + "benefit": "Avoids leaking secrets into logs.", + "compatibility_cost": "Existing log consumers lose raw values.", + "affected_cfs": [ + "CF-BND-001" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-CMDLET-135", + "baseline_behavior": "Cmdlets can require a transaction (-UseTransaction, SupportsTransactions) via an engine protection scope.", + "proposed_behavior": "Omit transaction support.", + "benefit": "Removes a Windows-only feature and engine scope handling.", + "compatibility_cost": "Cmdlets and scripts using transactions are unsupported.", + "affected_cfs": [ + "CF-LCY-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T34.json b/docs-out/.work/cf/T34.json new file mode 100644 index 0000000..3bc44f1 --- /dev/null +++ b/docs-out/.work/cf/T34.json @@ -0,0 +1,23 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-PROV-140", + "baseline_behavior": "Providers receive a mutable Context set by the engine and have no policy mediation layer.", + "proposed_behavior": "A mandatory policy mediation layer between engine and providers and an explicit per-call context object.", + "benefit": "Root confinement, rights and audit enforced outside provider code.", + "compatibility_cost": "Provider authors must target a new context type.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-PROV-141", + "baseline_behavior": "File-system drives are auto-mounted on lookup miss.", + "proposed_behavior": "Mount drives explicitly or from configuration.", + "benefit": "Predictable drive table; no hidden system probing.", + "compatibility_cost": "Scripts using unmounted drive letters need explicit mounts.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T35.json b/docs-out/.work/cf/T35.json new file mode 100644 index 0000000..af8c003 --- /dev/null +++ b/docs-out/.work/cf/T35.json @@ -0,0 +1,42 @@ +{ + "cfs": [ + { + "id": "CF-ETS-001", + "subject": "member resolution order", + "statement": "**Member resolution order (CF-ETS-001).** Looking up member *name* on a value `v` (wrapped as a `PSObject`) proceeds in this order and returns the first hit: (1) the reserved names `psbase`, `psadapted`, `psextended`, `psobject` and `pstypenames`, generated on demand; (2) **instance members** added to this object (for example by `Add-Member`); (3) **type-table (extended) members** merged over the object's type-name hierarchy, most specific type name winning; (4) **adapted members** from the adapter chosen for the object; (5) **base members**: the .NET members of the wrapped object, only when the chosen adapter is not already the .NET adapter. Member-name matching in the .NET adapter is case-insensitive (its binding flags include `IgnoreCase`). Type-table collection entries are marked \"replicate and clone when returning\" (a copy bound to the current object is handed out), adapted and base entries are not; the enumeration order across sources is `UNVERIFIED` (VB-038). Member lookup on a `PSMemberSet` owner searches the set's own members first, then the same collections.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152", + "src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMappedAdapter@L460-L538", + "src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041", + "src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMemberCollection@L261-L309" + ], + "owner": "03-runtime/extended-type-system.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-ETS-150", + "baseline_behavior": "Adapters exist for WMI, DirectoryEntry, DataRow/DataRowView, COM, CIM and XML nodes.", + "proposed_behavior": "Ship .NET, XML, PSObject, member-set and the third-party adapter bridge only.", + "benefit": "Smaller surface, fewer Windows-only dependencies.", + "compatibility_cost": "Scripts using those object kinds lose adapted members.", + "affected_cfs": [ + "CF-ETS-001" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-ETS-151", + "baseline_behavior": "Built-in type data is compiled into C# from types.ps1xml.", + "proposed_behavior": "Compile at build time and also publish a versioned data file.", + "benefit": "One source for tooling and a clean-room implementation.", + "compatibility_cost": "None observable.", + "affected_cfs": [ + "CF-ETS-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T36.json b/docs-out/.work/cf/T36.json new file mode 100644 index 0000000..26ed403 --- /dev/null +++ b/docs-out/.work/cf/T36.json @@ -0,0 +1,41 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-FMT-160", + "baseline_behavior": "Output is text-only through a LineOutput sink.", + "proposed_behavior": "Add an optional structured (typed record) sink next to the text sink.", + "benefit": "Machine consumers need not parse text.", + "compatibility_cost": "Additional sink contract to maintain.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-FMT-161", + "baseline_behavior": "Format data is loaded from an XML schema with a large loader family.", + "proposed_behavior": "Built-in views as compiled data and a small JSON/YAML definition for extensions.", + "benefit": "Smaller loader, simpler validation.", + "compatibility_cost": "Existing format.ps1xml files need conversion.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-HELP-165", + "baseline_behavior": "Updatable help downloads unsigned packages from a module-provided URI.", + "proposed_behavior": "Signed help packages with a catalog and offline mode, verified before install.", + "benefit": "Integrity of installed help.", + "compatibility_cost": "Help authors must sign packages.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-HELP-166", + "baseline_behavior": "Help is authored in MAML XML plus a DSC resource provider.", + "proposed_behavior": "Lighter structured format and comment-based help only.", + "benefit": "Simpler authoring and parsing.", + "compatibility_cost": "Existing MAML help needs conversion.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T37.json b/docs-out/.work/cf/T37.json new file mode 100644 index 0000000..401f935 --- /dev/null +++ b/docs-out/.work/cf/T37.json @@ -0,0 +1,43 @@ +{ + "cfs": [ + { + "id": "CF-MOD-001", + "subject": "module load, export and auto-load rules", + "statement": "**Module load, export and auto-load rules (CF-MOD-001).** (1) A module name without a path is searched in each `PSModulePath` entry in order (entries marked to skip the system folder are skipped; on Unix the folder name is matched case-insensitively): in an entry, the module base is `/`; its version sub-folders are tried first (the first whose version satisfies the constraints and equals its manifest's `ModuleVersion` is loaded); if none loads, the files `//` + extension are tried in the order `.psd1`, `.psm1`, `.cdxml`, `.ni.dll`, `.dll`, `.exe` (a name that already carries a module extension is tried as given); the first successful load ends the search and its exports are added to the analysis cache. (2) A manifest is read as data and its keys are checked against the known key list (\u00a74); the processing of the root module, nested modules, required modules (version-constrained; a cyclic-requirement check exists), required assemblies, scripts to process, types and formats follows, in an order that is `UNVERIFIED` (VB-046); how the manifest export lists combine with a script module's own exports is `UNVERIFIED` (VB-046). (3) A module already loaded that satisfies the requested constraints is not loaded again unless `-Force`; only its members are re-imported with the requested prefix and options. (4) Each module has its own session state; exported commands are copied into the importer's session state. (5) `Export-ModuleMember` selects exports by wildcard patterns per category (functions, cmdlets, variables, aliases), skipping `AllScope` items and built-in variable names. (6) Import applies `-Prefix`/`DefaultCommandPrefix`, `-NoClobber`, visibility and the verb/noun checks, in that order: cmdlets, aliases, functions, variables. (7) **Auto-loading** happens during command lookup: a module-qualified name loads exactly that module; any other unknown command name is looked up in the analysis cache of every module file on `PSModulePath` and the first module exporting a command of that exact name is imported (`$PSModuleAutoLoadingPreference`: `None`, `ModuleQualified`, `All`). (8) Removal of a module (`RemoveModule`) is `UNVERIFIED` here (VB-046).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingModulePath@L343-L423", + "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingExtensions@L5422-L5518", + "src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.ExportModuleMembers@L1529-L1689", + "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.ImportModuleMembers@L7022-L7251", + "src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCache.GetExportedCommands@L44-L96" + ], + "owner": "03-runtime/modules.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-MOD-170", + "baseline_behavior": "PSModulePath entries have equal trust and auto-load picks the first module exporting a name.", + "proposed_behavior": "Module search path entries carry trust levels; auto-load only from trusted entries unless policy allows.", + "benefit": "Prevents shadowing through writable low-trust folders.", + "compatibility_cost": "Auto-load from user-writable folders needs explicit policy.", + "affected_cfs": [ + "CF-MOD-001" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-MOD-171", + "baseline_behavior": "Windows PowerShell compatibility imports, .cdxml CIM modules and DSC resource exports are supported.", + "proposed_behavior": "Omit in the first version.", + "benefit": "Smaller scope; removes Windows-only machinery.", + "compatibility_cost": "Those modules cannot be imported.", + "affected_cfs": [ + "CF-MOD-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T38.json b/docs-out/.work/cf/T38.json new file mode 100644 index 0000000..48da22b --- /dev/null +++ b/docs-out/.work/cf/T38.json @@ -0,0 +1,31 @@ +{ + "cfs": [ + { + "id": "CF-ERR-001", + "subject": "terminating/non-terminating errors; try/catch/finally/trap ordering", + "statement": "**Terminating versus non-terminating errors; try/catch/finally/trap ordering (CF-ERR-001).** (1) A *non-terminating* error is an `ErrorRecord` written to the error stream by a command (`WriteError`); it does not interrupt the command or the pipeline, sets the pipeline's failed state and `$?` to false, and is subject to the command's `ErrorAction` (`Ignore`: dropped, not recorded; `SilentlyContinue`: recorded in `$Error`/`-ErrorVariable` but not shown; `Continue`: recorded and written; `Inquire`: ask, then as `Continue` or stop; `Stop`: converted to a terminating `ActionPreferenceStopException`). (2) A *terminating* error is an exception: `ThrowTerminatingError` from a cmdlet (wrapped as `CmdletInvocationException`), `throw`, a runtime error in an expression, or `ActionPreferenceStop`. It stops the current *statement* and, unless handled, the command, and is recorded as the pipeline's first terminating error (later ones are only logged). (3) In **script code every statement is guarded**: when a statement throws a non-flow-control exception the engine consults `$ErrorActionPreference` (default `Continue`): `SilentlyContinue`/`Ignore` resume with the next statement; `Continue` reports the error record and resumes with the next statement \u2014 except that an exception produced by a `throw` statement is rethrown when no trap is present; `Stop` rethrows (terminates the enclosing scope and propagates); `Inquire` prompts. (4) **Handler order** for one thrown exception: innermost enclosing `try` is searched first; its `catch` clauses are tried in source order with the first matching type winning (a catch without a type matches any `RuntimeException`; `PipelineStoppedException` is never caught by a type-less catch); `finally` runs after the try/catch block completes, on normal exit, after a caught exception, and while an unhandled exception propagates; if no `catch` in the `try` matches, the exception propagates to the enclosing `try`, then to `trap` handlers of the enclosing blocks (a trap matches by the exception type, trying the inner exception before the outer one), then to the statement-level preference rule of (3). A `trap` runs in a new scope with `$_` set to the error record; `continue` in a trap means *SilentlyContinue*, `break` means *Stop*; otherwise the action is the current preference. (5) Flow control (`break`, `continue`, `return`, `exit`) travels as exceptions that are never caught by `catch` or traps and are rethrown by the statement guard. (6) `$?` is true after a statement that completes without error and false after a statement in which an error (terminating-handled or non-terminating) occurred; the pipeline-level failed state also covers errors written by commands.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788", + "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968", + "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.RecordFailure@L1455-L1498", + "src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.ManageInvocationException@L848-L947" + ], + "owner": "03-runtime/errors-and-diagnostics.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-ERR-180", + "baseline_behavior": "Errors are only on the user error stream and honour -ErrorAction/$ErrorActionPreference, including suppression.", + "proposed_behavior": "A separate audit stream for policy decisions and denials that user preferences cannot suppress.", + "benefit": "Audit integrity.", + "compatibility_cost": "Additional stream to consume.", + "affected_cfs": [ + "CF-ERR-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T39.json b/docs-out/.work/cf/T39.json new file mode 100644 index 0000000..b172479 --- /dev/null +++ b/docs-out/.work/cf/T39.json @@ -0,0 +1,59 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-JOB-190", + "baseline_behavior": "Background jobs start a separate pwsh process and parallel tasks use runspaces created from the session; policy inheritance is not enforced in the code read.", + "proposed_behavior": "Child processes and parallel runspaces inherit policy, language mode and visibility as a hard rule, with per-job quotas.", + "benefit": "Prevents escape from a restricted session through jobs.", + "compatibility_cost": "Jobs cannot be more privileged than the parent.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-JOB-191", + "baseline_behavior": "ThrottlingJob, ContainerParentJob and a job source-adapter registry exist.", + "proposed_behavior": "One local job type plus the parallel pool.", + "benefit": "Smaller job model.", + "compatibility_cost": "Third-party job sources unsupported initially.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-DBG-195", + "baseline_behavior": "Debugger enable/attach is not a distinct audited privilege.", + "proposed_behavior": "Policy-controlled, audited debugger attach; prompt runs at the lowest trust level on the stack.", + "benefit": "Controls a code-execution channel.", + "compatibility_cost": "Debug sessions need explicit permission.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-DBG-196", + "baseline_behavior": "Nested debugging of jobs and remote runspaces exists.", + "proposed_behavior": "Omit in the first version.", + "benefit": "Smaller debugger.", + "compatibility_cost": "No job or remote debugging initially.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-EVT-198", + "baseline_behavior": "Received-event and action queues are unbounded.", + "proposed_behavior": "Bounded queues with policy-defined drop rules.", + "benefit": "Resource safety.", + "compatibility_cost": "Events may be dropped under load.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-EVT-199", + "baseline_behavior": "Forwarded events and WMI/CIM event sources are supported.", + "proposed_behavior": "Omit in the first version.", + "benefit": "Smaller eventing surface.", + "compatibility_cost": "No event forwarding initially.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/claims/T3.jsonl b/docs-out/.work/claims/T3.jsonl new file mode 100644 index 0000000..73b079a --- /dev/null +++ b/docs-out/.work/claims/T3.jsonl @@ -0,0 +1,43 @@ +{"id": "CLM-PIP-0001", "subject_key": "pipeline.pipe.default_out_buffer_count", "predicate": "default OutBufferCount of a Pipe", "value": "0", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Pipe.cs::Pipe.OutBufferCount@L110-L110"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0002", "subject_key": "pipeline.pipe.add_order", "predicate": "order of sinks checked by Pipe.AddToPipe", "value": "nested PipelineProcessor, result collection, result list, external writer, queue", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0003", "subject_key": "pipeline.pipe.retrieve_order", "predicate": "order of sources read by Pipe.Retrieve", "value": "queue, enumerator, external reader", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Pipe.cs::Pipe.Retrieve@L541-L601"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0004", "subject_key": "pipeline.start.binding_before_begin", "predicate": "all commands are prepared (bound) before any begin phase", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147"], "doc": "RT-PIPELINE-001", "cf": "CF-LCY-001"} +{"id": "CLM-PIP-0005", "subject_key": "pipeline.first_error", "predicate": "later terminating failures replace the first recorded error", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.RecordFailure@L1455-L1498"], "doc": "RT-PIPELINE-001", "cf": "CF-LCY-001"} +{"id": "CLM-PIP-0006", "subject_key": "pipeline.thread.stack_size_bytes", "predicate": "DefaultPipelineStackSize", "value": "10000000", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline@L21-L1143"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0007", "subject_key": "pipeline.state.pipeline_state_members", "predicate": "PipelineState members in order", "value": "NotStarted, Running, Stopping, Stopped, Completed, Failed, Disconnected", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/Pipeline.cs::PipelineState@L132-L162"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0008", "subject_key": "pipeline.concurrency.nested_must_be_sync", "predicate": "nested pipelines must be invoked synchronously", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.DoConcurrentCheck@L617-L700"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-DSC-0009", "subject_key": "discovery.order", "predicate": "search order of CommandSearcher", "value": "alias, function, cmdlet, qualified provider path, rooted path, PATH search, relative path", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239"], "doc": "RT-DISCOVERY-001", "cf": "CF-DSC-001"} +{"id": "CLM-DSC-0010", "subject_key": "discovery.get_prefix_retry", "predicate": "name without dash or backslash is retried with Get- prefix", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.TryNormalSearch@L886-L949"], "doc": "RT-DISCOVERY-001", "cf": "CF-DSC-001"} +{"id": "CLM-DSC-0011", "subject_key": "discovery.command_types.workflow_member", "predicate": "CommandTypes has a Workflow member", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommandInfo.cs::CommandTypes@L23-L76"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-DSC-0012", "subject_key": "discovery.native.process_start_phase", "predicate": "native process is started in", "value": "Prepare (before any begin phase)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Prepare@L557-L579"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-DSC-0013", "subject_key": "discovery.native.arg_passing_default", "predicate": "default value of the native argument passing variable", "value": "Windows on Windows, Standard on Unix", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.GetPassingStyle@L4675-L4682"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-DSC-0014", "subject_key": "discovery.native.exit_code_error_requires_preference", "predicate": "non-zero exit code raises an error only when PSNativeCommandUseErrorActionPreference is true", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-BND-0015", "subject_key": "binding.positional.attempt_order", "predicate": "positional binding attempts per argument", "value": "default set no coercion, all sets no coercion, default set coercion, all sets coercion", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController.BindPositionalParameters@L699-L871"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-BND-0016", "subject_key": "binding.pipeline.pass_order", "predicate": "pipeline binding passes", "value": "ValueFromPipeline no coercion, ByPropertyName no coercion, ValueFromPipeline coercion, ByPropertyName coercion", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindPipelineParametersPrivate@L3255-L3347"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-BND-0017", "subject_key": "binding.parameter_sets.max", "predicate": "maximum parameter sets per command", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.MaxParameterSetCount@L171-L171"], "doc": "RT-BIND-001", "cf": null} +{"id": "CLM-BND-0018", "subject_key": "binding.name_match", "predicate": "parameter name matching", "value": "case-insensitive unambiguous prefix over names and aliases; exact match wins", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.GetMatchingParameter@L445-L578"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-BND-0019", "subject_key": "binding.value.order", "predicate": "steps when binding one value", "value": "transformation, coercion, PSTypeName check, validation, mandatory null check, set", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L329-L667"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-CMD-0020", "subject_key": "cmdlet.common_parameters.count", "predicate": "number of common parameters on CommonParameters", "value": "12", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommonCommandParameters.cs::CommonParameters@L13-L283"], "doc": "RT-CMDLET-001", "cf": null} +{"id": "CLM-CMD-0021", "subject_key": "cmdlet.write_permission", "predicate": "writes allowed only inside AllowThisCommandToWrite windows on the recorded thread", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.ThrowIfWriteNotPermitted@L2325-L2341"], "doc": "RT-CMDLET-001", "cf": null} +{"id": "CLM-CMD-0022", "subject_key": "cmdlet.shouldprocess.order", "predicate": "ShouldProcess decision order", "value": "remembered answer, WhatIf, impact threshold, prompt", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.DoShouldProcess@L1515-L1621"], "doc": "RT-CMDLET-001", "cf": null} +{"id": "CLM-PRV-0023", "subject_key": "provider.instance_per_operation", "predicate": "provider instances are created per operation", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo.CreateInstance@L546-L638"], "doc": "RT-PROVIDER-001", "cf": null} +{"id": "CLM-PRV-0024", "subject_key": "provider.hierarchy", "predicate": "provider class hierarchy", "value": "CmdletProvider, DriveCmdletProvider, ItemCmdletProvider, ContainerCmdletProvider, NavigationCmdletProvider", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo@L22-L687"], "doc": "RT-PROVIDER-001", "cf": null} +{"id": "CLM-ETS-0025", "subject_key": "ets.adapter_lookup", "predicate": "type-table adapter lookup key", "value": "exact full type name", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/TypeTable.cs::TypeTable.GetTypeAdapter@L4064-L4099"], "doc": "RT-ETS-001", "cf": "CF-ETS-001"} +{"id": "CLM-ETS-0026", "subject_key": "ets.member_order", "predicate": "member lookup order", "value": "reserved names, instance, type table, adapter, base", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152"], "doc": "RT-ETS-001", "cf": "CF-ETS-001"} +{"id": "CLM-ETS-0027", "subject_key": "ets.typetable.merge_direction", "predicate": "type-table members merged from", "value": "least specific to most specific type name", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041"], "doc": "RT-ETS-001", "cf": "CF-ETS-001"} +{"id": "CLM-FMT-0028", "subject_key": "format.table_cache_window_ms", "predicate": "auto-width table caching window", "value": "300", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394"], "doc": "RT-FORMAT-001", "cf": null} +{"id": "CLM-FMT-0029", "subject_key": "format.fallback_shape", "predicate": "shape when no view and property count at or below table threshold", "value": "Table", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs::DisplayDataQuery.GetShapeFromPropertyCount@L333-L339"], "doc": "RT-FORMAT-001", "cf": null} +{"id": "CLM-HLP-0030", "subject_key": "help.provider_order", "predicate": "help provider chain order", "value": "alias, script command, command, provider, class, help file, default", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.InitializeHelpProviders@L563-L593"], "doc": "RT-HELP-001", "cf": null} +{"id": "CLM-MOD-0031", "subject_key": "modules.extension_order", "predicate": "implicit module file extension order", "value": ".psd1, .psm1, .cdxml, .ni.dll, .dll, .exe", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.PSModuleExtensions@L896-L905"], "doc": "RT-MODULES-001", "cf": "CF-MOD-001"} +{"id": "CLM-MOD-0032", "subject_key": "modules.version_folder_rule", "predicate": "version folder accepted only if manifest ModuleVersion equals folder version", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingMultiVersionModuleBase@L433-L478"], "doc": "RT-MODULES-001", "cf": "CF-MOD-001"} +{"id": "CLM-MOD-0033", "subject_key": "modules.analysis_cache_flush_delay_ms", "predicate": "delay before the analysis cache is serialized", "value": "10000", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCacheData@L645-L1131"], "doc": "RT-MODULES-001", "cf": null} +{"id": "CLM-ERR-0034", "subject_key": "errors.dollar_error_max", "predicate": "maximum entries kept in $Error", "value": "256", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L818-L858"], "doc": "RT-ERRORS-001", "cf": null} +{"id": "CLM-ERR-0035", "subject_key": "errors.error_category.count", "predicate": "number of ErrorCategory members", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ErrorPackage.cs::ErrorCategory@L28-L189"], "doc": "RT-ERRORS-001", "cf": null} +{"id": "CLM-ERR-0036", "subject_key": "errors.question_mark", "predicate": "$? is read-only and all-scope", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/QuestionMarkVariable.cs::QuestionMarkVariable@L9-L42"], "doc": "RT-ERRORS-001", "cf": null} +{"id": "CLM-JOB-0037", "subject_key": "jobs.foreach_parallel.default_throttle", "predicate": "default ThrottleLimit of ForEach-Object -Parallel", "value": "5", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/InternalCommands.cs::ForEachObjectCommand@L65-L1271"], "doc": "RT-JOBS-001", "cf": null} +{"id": "CLM-JOB-0038", "subject_key": "jobs.job_states.count", "predicate": "number of JobState members", "value": "11", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/remoting/client/Job.cs::JobState@L31-L90"], "doc": "RT-JOBS-001", "cf": null} +{"id": "CLM-JOB-0039", "subject_key": "jobs.background.child_args", "predicate": "child pwsh arguments of a background job", "value": "-s -NoLogo -NoProfile", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs::PowerShellProcessInstance@L16-L292"], "doc": "RT-JOBS-001", "cf": null} +{"id": "CLM-DBG-0040", "subject_key": "debugger.resume_actions", "predicate": "DebuggerResumeAction members", "value": "Continue, StepInto, StepOut, StepOver, Stop", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerResumeAction@L31-L53"], "doc": "RT-DEBUGGER-001", "cf": null} +{"id": "CLM-DBG-0041", "subject_key": "debugger.prompt_commands", "predicate": "debugger prompt commands", "value": "continue c, stepInto s, stepOver v, stepOut o, quit q, detach d, k, list l, h", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerCommandProcessor@L5085-L5419"], "doc": "RT-DEBUGGER-001", "cf": null} +{"id": "CLM-EVT-0042", "subject_key": "events.reserved_prefix", "predicate": "source identifiers reserved for the engine start with", "value": "PowerShell.", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewSubscriber@L645-L796"], "doc": "RT-EVENTING-001", "cf": null} +{"id": "CLM-EVT-0043", "subject_key": "events.action_throttle", "predicate": "actions run per pulse", "value": "1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager@L321-L1546"], "doc": "RT-EVENTING-001", "cf": null} diff --git a/docs-out/.work/files-ledger.csv b/docs-out/.work/files-ledger.csv index 955cdc4..b252fd7 100644 --- a/docs-out/.work/files-ledger.csv +++ b/docs-out/.work/files-ledger.csv @@ -1547,10 +1547,10 @@ src/System.Management.Automation/FormatAndOutput/DefaultFormatters/PowerShellCor src/System.Management.Automation/FormatAndOutput/DefaultFormatters/PowerShellTrace_format_ps1xml.cs,46,B,,,unread src/System.Management.Automation/FormatAndOutput/DefaultFormatters/Registry_format_ps1xml.cs,55,B,,,unread src/System.Management.Automation/FormatAndOutput/DefaultFormatters/WSMan_Format_ps1xml.cs,126,B,,,unread -src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs,407,B,,,unread +src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs,407,B,1-400,RT-FORMAT-001,partial src/System.Management.Automation/FormatAndOutput/common/BaseFormattingCommand.cs,899,B,,,unread src/System.Management.Automation/FormatAndOutput/common/BaseFormattingCommandParameters.cs,489,B,,,unread -src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs,1395,B,,,unread +src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs,1395,B,16-166;256-400,RT-FORMAT-001,partial src/System.Management.Automation/FormatAndOutput/common/ColumnWidthManager.cs,208,B,,,unread src/System.Management.Automation/FormatAndOutput/common/ComplexWriter.cs,829,B,,,unread src/System.Management.Automation/FormatAndOutput/common/FormatGroupManager.cs,123,B,,,unread @@ -1560,13 +1560,13 @@ src/System.Management.Automation/FormatAndOutput/common/FormatViewGenerator_Comp src/System.Management.Automation/FormatAndOutput/common/FormatViewGenerator_List.cs,236,B,,,unread src/System.Management.Automation/FormatAndOutput/common/FormatViewGenerator_Table.cs,493,B,,,unread src/System.Management.Automation/FormatAndOutput/common/FormatViewGenerator_Wide.cs,197,B,,,unread -src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs,685,B,,,unread +src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs,685,B,91-215;399-467,RT-FORMAT-001,partial src/System.Management.Automation/FormatAndOutput/common/FormatXMLWriter.cs,454,B,,,unread src/System.Management.Automation/FormatAndOutput/common/FormattingObjects.cs,395,B,,,unread src/System.Management.Automation/FormatAndOutput/common/FormattingObjectsDeserializer.cs,700,B,,,unread src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs,580,B,,,unread src/System.Management.Automation/FormatAndOutput/common/ListWriter.cs,288,B,,,unread -src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs,327,B,,,unread +src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs,327,B,19-326,RT-FORMAT-001,partial src/System.Management.Automation/FormatAndOutput/common/OutputQueue.cs,333,B,,,unread src/System.Management.Automation/FormatAndOutput/common/PSStyle.cs,916,B,,,unread src/System.Management.Automation/FormatAndOutput/common/StringDecorated.cs,189,B,,,unread @@ -1583,7 +1583,7 @@ src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/displayD src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/displayDescriptionData_Wide.cs,318,B,,,unread src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/displayResourceManagerCache.cs,225,B,,,unread src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataManager.cs,570,B,,,unread -src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs,673,B,,,unread +src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs,673,B,333-586,RT-FORMAT-001,partial src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataXmlLoader.cs,2235,B,,,unread src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataXmlLoader_Complex.cs,653,B,,,unread src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataXmlLoader_List.cs,315,B,,,unread @@ -1594,9 +1594,9 @@ src/System.Management.Automation/FormatAndOutput/common/Utilities/MshObjectUtil. src/System.Management.Automation/FormatAndOutput/common/Utilities/MshParameter.cs,529,B,,,unread src/System.Management.Automation/FormatAndOutput/common/Utilities/MshParameterAssociation.cs,227,B,,,unread src/System.Management.Automation/FormatAndOutput/common/Utilities/Mshexpression.cs,379,B,,,unread -src/System.Management.Automation/FormatAndOutput/format-default/format-default.cs,39,B,,,unread +src/System.Management.Automation/FormatAndOutput/format-default/format-default.cs,39,B,1-39,RT-FORMAT-001,read src/System.Management.Automation/FormatAndOutput/out-console/ConsoleLineOutput.cs,549,B,,,unread -src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs,212,B,,,unread +src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs,212,B,36-170,RT-FORMAT-001,partial src/System.Management.Automation/FormatAndOutput/out-textInterface/OutTextInterface.cs,92,B,,,unread src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.cs,136,B,,,unread src/System.Management.Automation/cimSupport/cmdletization/EnumWriter.cs,71,B,,,unread @@ -1616,32 +1616,32 @@ src/System.Management.Automation/engine/ArgumentToVersionTransformationAttribute src/System.Management.Automation/engine/ArgumentTypeConverterAttribute.cs,208,B,,,unread src/System.Management.Automation/engine/AsyncByteStreamTransfer.cs,83,B,,,unread src/System.Management.Automation/engine/Attributes.cs,2364,B,,,unread -src/System.Management.Automation/engine/AutomationEngine.cs,129,B,,,unread +src/System.Management.Automation/engine/AutomationEngine.cs,129,B,56-129,RT-PIPELINE-001,partial src/System.Management.Automation/engine/AutomationNull.cs,33,B,,,unread src/System.Management.Automation/engine/BytePipe.cs,113,B,,,unread src/System.Management.Automation/engine/ChildrenCmdletProviderInterfaces.cs,786,B,,,unread src/System.Management.Automation/engine/CmdletFamilyProviderInterfaces.cs,109,B,,,unread src/System.Management.Automation/engine/CmdletInfo.cs,558,B,,,unread -src/System.Management.Automation/engine/CmdletParameterBinderController.cs,4653,A,,,unread +src/System.Management.Automation/engine/CmdletParameterBinderController.cs,4653,A,104-470;854-990;1198-1320;1463-1580;1595-1735;1756-2000;2532-2700;2752-2905;3147-3640,RT-BIND-001,partial src/System.Management.Automation/engine/CodeMethods.cs,135,B,,,unread -src/System.Management.Automation/engine/CommandBase.cs,604,B,,,unread -src/System.Management.Automation/engine/CommandDiscovery.cs,1646,A,,,unread -src/System.Management.Automation/engine/CommandInfo.cs,984,B,,,unread +src/System.Management.Automation/engine/CommandBase.cs,604,B,250-270;300-400,RT-CMDLET-001;RT-ERRORS-001,partial +src/System.Management.Automation/engine/CommandDiscovery.cs,1646,A,276-660;862-1260;1284-1464,RT-DISCOVERY-001,partial +src/System.Management.Automation/engine/CommandInfo.cs,984,B,20-80,RT-DISCOVERY-001,partial src/System.Management.Automation/engine/CommandMetadata.cs,1551,A,,,unread src/System.Management.Automation/engine/CommandParameter.cs,230,B,,,unread src/System.Management.Automation/engine/CommandPathSearch.cs,596,B,,,unread -src/System.Management.Automation/engine/CommandProcessor.cs,869,A,,,unread -src/System.Management.Automation/engine/CommandProcessorBase.cs,1044,A,,,unread -src/System.Management.Automation/engine/CommandSearcher.cs,1733,A,,,unread +src/System.Management.Automation/engine/CommandProcessor.cs,869,A,1-869,RT-BIND-001;RT-PIPELINE-001,read +src/System.Management.Automation/engine/CommandProcessorBase.cs,1044,A,20-1044,RT-CMDLET-001;RT-PIPELINE-001,partial +src/System.Management.Automation/engine/CommandSearcher.cs,1733,A,22-1100;1322-1640,RT-DISCOVERY-001,partial src/System.Management.Automation/engine/CommonCommandParameters.cs,284,B,,,unread src/System.Management.Automation/engine/CompiledCommandParameter.cs,683,A,,,unread src/System.Management.Automation/engine/ConfigurationInfo.cs,172,B,,,unread src/System.Management.Automation/engine/ContentCmdletProviderInterfaces.cs,594,B,,,unread -src/System.Management.Automation/engine/CoreAdapter.cs,6259,B,,,unread +src/System.Management.Automation/engine/CoreAdapter.cs,6259,B,38-330;2616-2700,RT-ETS-001,partial src/System.Management.Automation/engine/Credential.cs,404,B,,,unread src/System.Management.Automation/engine/CultureVariable.cs,59,B,,,unread src/System.Management.Automation/engine/DataStoreAdapter.cs,832,B,,,unread -src/System.Management.Automation/engine/DataStoreAdapterProvider.cs,688,B,,,unread +src/System.Management.Automation/engine/DataStoreAdapterProvider.cs,688,B,546-638,RT-PROVIDER-001,partial src/System.Management.Automation/engine/DefaultCommandRuntime.cs,246,B,,,unread src/System.Management.Automation/engine/DriveInterfaces.cs,394,B,,,unread src/System.Management.Automation/engine/DriveNames.cs,37,B,,,unread @@ -1651,8 +1651,8 @@ src/System.Management.Automation/engine/EngineIntrinsics.cs,113,B,,,unread src/System.Management.Automation/engine/EnumExpressionEvaluator.cs,644,B,,,unread src/System.Management.Automation/engine/EnumMinimumDisambiguation.cs,127,B,,,unread src/System.Management.Automation/engine/ErrorPackage.cs,1837,B,,,unread -src/System.Management.Automation/engine/EventManager.cs,2648,B,,,unread -src/System.Management.Automation/engine/ExecutionContext.cs,1711,A,100-145;275-300;521-560;862-885,RT-CONTEXT-001,partial +src/System.Management.Automation/engine/EventManager.cs,2648,B,22-140;326-392;510-572;645-796;957-1137;1222-1295;1821-1850,RT-EVENTING-001,partial +src/System.Management.Automation/engine/ExecutionContext.cs,1711,A,100-145;275-300;521-560;818-858;862-885,RT-CONTEXT-001;RT-ERRORS-001,partial src/System.Management.Automation/engine/ExtendedTypeSystemException.cs,641,B,,,unread src/System.Management.Automation/engine/ExternalScriptInfo.cs,633,B,,,unread src/System.Management.Automation/engine/ExtraAdapter.cs,385,B,,,unread @@ -1661,26 +1661,26 @@ src/System.Management.Automation/engine/FunctionInfo.cs,497,B,,,unread src/System.Management.Automation/engine/GetCommandCommand.cs,1743,B,,,unread src/System.Management.Automation/engine/ICommandRuntime.cs,622,B,,,unread src/System.Management.Automation/engine/InformationRecord.cs,264,B,,,unread -src/System.Management.Automation/engine/InitialSessionState.cs,5674,B,,,unread -src/System.Management.Automation/engine/InternalCommands.cs,2752,B,,,unread +src/System.Management.Automation/engine/InitialSessionState.cs,5674,B,4655-4685;5515-5535,RT-DISCOVERY-001;RT-PROVIDER-001,partial +src/System.Management.Automation/engine/InternalCommands.cs,2752,B,240-271;377-470,RT-JOBS-001,partial src/System.Management.Automation/engine/InvocationInfo.cs,537,B,,,unread src/System.Management.Automation/engine/ItemCmdletProviderInterfaces.cs,2107,B,,,unread src/System.Management.Automation/engine/LanguagePrimitives.cs,5888,A,449-560;599-865;980-1060;1670-1770;2882-3074;3096-3135;3322-3400;3581-3660;4468-4520;4860-4895;4966-5120;5641-5848,LANG-CONV-001;LANG-TYPES-001,partial src/System.Management.Automation/engine/ManagementObjectAdapter.cs,1264,B,,,unread -src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs,717,A,,,unread +src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs,717,A,78-160;164-176;278-345;445-580,RT-BIND-001,partial src/System.Management.Automation/engine/MinishellParameterBinderController.cs,384,A,,,unread src/System.Management.Automation/engine/MshCmdlet.cs,1030,B,,,unread -src/System.Management.Automation/engine/MshCommandRuntime.cs,3830,B,,,unread -src/System.Management.Automation/engine/MshMemberInfo.cs,5157,B,,,unread -src/System.Management.Automation/engine/MshObject.cs,2643,B,,,unread +src/System.Management.Automation/engine/MshCommandRuntime.cs,3830,B,164-292;667-742;1446-1655;2325-2400;2648-2730;2858-3010;3516-3600,RT-CMDLET-001;RT-ERRORS-001,partial +src/System.Management.Automation/engine/MshMemberInfo.cs,5157,B,33-150;4503-4560;4799-4860,RT-ETS-001,partial +src/System.Management.Automation/engine/MshObject.cs,2643,B,245-330;426-545;1938-2116,RT-ETS-001,partial src/System.Management.Automation/engine/MshObjectTypeDescriptor.cs,769,B,,,unread src/System.Management.Automation/engine/MshReference.cs,90,B,,,unread src/System.Management.Automation/engine/MshSecurityException.cs,111,B,,,unread src/System.Management.Automation/engine/MshSnapinQualifiedName.cs,134,B,,,unread src/System.Management.Automation/engine/NativeCommand.cs,36,B,,,unread -src/System.Management.Automation/engine/NativeCommandParameterBinder.cs,520,A,,,unread +src/System.Management.Automation/engine/NativeCommandParameterBinder.cs,520,A,58-470,RT-DISCOVERY-001,partial src/System.Management.Automation/engine/NativeCommandParameterBinderController.cs,106,A,,,unread -src/System.Management.Automation/engine/NativeCommandProcessor.cs,2651,A,,,unread +src/System.Management.Automation/engine/NativeCommandProcessor.cs,2651,A,197-232;557-606;715-967;1082-1240;1449-1514;1586-1884,RT-DISCOVERY-001,partial src/System.Management.Automation/engine/NullString.cs,45,B,,,unread src/System.Management.Automation/engine/ObjectEventRegistrationBase.cs,232,B,,,unread src/System.Management.Automation/engine/OrderedHashtable.cs,236,B,,,unread @@ -1688,14 +1688,14 @@ src/System.Management.Automation/engine/PSClassInfo.cs,86,B,,,unread src/System.Management.Automation/engine/PSClassSearcher.cs,329,B,,,unread src/System.Management.Automation/engine/PSConfiguration.cs,794,B,,,unread src/System.Management.Automation/engine/PSVersionInfo.cs,1038,B,,,unread -src/System.Management.Automation/engine/ParameterBinderBase.cs,2067,A,,,unread -src/System.Management.Automation/engine/ParameterBinderController.cs,1326,A,,,unread +src/System.Management.Automation/engine/ParameterBinderBase.cs,2067,A,329-667,RT-BIND-001,partial +src/System.Management.Automation/engine/ParameterBinderController.cs,1326,A,133-330;514-870;919-1190,RT-BIND-001,partial src/System.Management.Automation/engine/ParameterInfo.cs,146,B,,,unread src/System.Management.Automation/engine/ParameterSetInfo.cs,345,B,,,unread src/System.Management.Automation/engine/ParameterSetPromptingData.cs,62,B,,,unread src/System.Management.Automation/engine/ParameterSetSpecificMetadata.cs,214,B,,,unread src/System.Management.Automation/engine/PathInterfaces.cs,1414,B,,,unread -src/System.Management.Automation/engine/Pipe.cs,622,A,,,unread +src/System.Management.Automation/engine/Pipe.cs,622,A,1-622,RT-PIPELINE-001,read src/System.Management.Automation/engine/PositionalCommandParameter.cs,27,B,,,unread src/System.Management.Automation/engine/PowerShellStreamType.cs,59,B,,,unread src/System.Management.Automation/engine/ProcessCodeMethods.cs,72,B,,,unread @@ -1706,7 +1706,7 @@ src/System.Management.Automation/engine/ProviderNames.cs,132,B,,,unread src/System.Management.Automation/engine/ProxyCommand.cs,503,B,,,unread src/System.Management.Automation/engine/PseudoParameterBinder.cs,136,A,,,unread src/System.Management.Automation/engine/PseudoParameters.cs,246,B,,,unread -src/System.Management.Automation/engine/QuestionMarkVariable.cs,43,B,,,unread +src/System.Management.Automation/engine/QuestionMarkVariable.cs,43,B,1-45,RT-ERRORS-001,read src/System.Management.Automation/engine/ReflectionParameterBinder.cs,322,A,,,unread src/System.Management.Automation/engine/ScopedItemSearcher.cs,487,A,15-297,LANG-SCOPE-001,partial src/System.Management.Automation/engine/ScriptCommand.cs,16,A,,,unread @@ -1720,14 +1720,14 @@ src/System.Management.Automation/engine/SessionStateAliasAPIs.cs,585,B,,,unread src/System.Management.Automation/engine/SessionStateCmdletAPIs.cs,326,B,,,unread src/System.Management.Automation/engine/SessionStateContainer.cs,4897,B,,,unread src/System.Management.Automation/engine/SessionStateContent.cs,1035,B,,,unread -src/System.Management.Automation/engine/SessionStateDriveAPIs.cs,1436,B,,,unread +src/System.Management.Automation/engine/SessionStateDriveAPIs.cs,1436,B,64-232;410-477,RT-PROVIDER-001,partial src/System.Management.Automation/engine/SessionStateDynamicProperty.cs,2176,B,,,unread src/System.Management.Automation/engine/SessionStateFunctionAPIs.cs,791,B,,,unread src/System.Management.Automation/engine/SessionStateItem.cs,1377,B,,,unread src/System.Management.Automation/engine/SessionStateLocationAPIs.cs,1114,B,,,unread src/System.Management.Automation/engine/SessionStateNavigation.cs,1750,B,,,unread src/System.Management.Automation/engine/SessionStateProperty.cs,1118,B,,,unread -src/System.Management.Automation/engine/SessionStateProviderAPIs.cs,1556,B,,,unread +src/System.Management.Automation/engine/SessionStateProviderAPIs.cs,1556,B,253-297;953-1070,RT-PROVIDER-001,partial src/System.Management.Automation/engine/SessionStatePublic.cs,374,B,,,unread src/System.Management.Automation/engine/SessionStateScope.cs,2013,A,27-110;293-345;401-520;661-700;1736-1790,LANG-SCOPE-001,partial src/System.Management.Automation/engine/SessionStateScopeAPIs.cs,326,A,200-300,LANG-SCOPE-001,partial @@ -1740,7 +1740,7 @@ src/System.Management.Automation/engine/ShellVariable.cs,916,A,,,unread src/System.Management.Automation/engine/SpecialVariables.cs,426,A,320-353;401-425,LANG-SCOPE-001,partial src/System.Management.Automation/engine/ThirdPartyAdapter.cs,365,B,,,unread src/System.Management.Automation/engine/TypeMetadata.cs,1567,B,,,unread -src/System.Management.Automation/engine/TypeTable.cs,4779,B,,,unread +src/System.Management.Automation/engine/TypeTable.cs,4779,B,3965-4110,RT-ETS-001,partial src/System.Management.Automation/engine/TypeTable_GetEvent_Types_Ps1Xml.cs,201,B,,,unread src/System.Management.Automation/engine/TypeTable_TypesV3_Ps1Xml.cs,437,B,,,unread src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs,9139,B,,,unread @@ -1749,8 +1749,8 @@ src/System.Management.Automation/engine/Utils.cs,1866,B,,,unread src/System.Management.Automation/engine/VariableAttributeCollection.cs,160,A,,,unread src/System.Management.Automation/engine/VariableInterfaces.cs,448,A,,,unread src/System.Management.Automation/engine/VariablePath.cs,322,A,1-322,LANG-SCOPE-001,read -src/System.Management.Automation/engine/cmdlet.cs,1855,B,,,unread -src/System.Management.Automation/engine/pipeline.cs,1668,A,,,unread +src/System.Management.Automation/engine/cmdlet.cs,1855,B,120-215;1738-1770,RT-CMDLET-001,partial +src/System.Management.Automation/engine/pipeline.cs,1668,A,1-1668,RT-ERRORS-001;RT-PIPELINE-001,read src/System.Management.Automation/engine/regex.cs,1384,B,,,unread src/System.Management.Automation/engine/scriptparameterbinder.cs,170,A,,,unread src/System.Management.Automation/engine/scriptparameterbindercontroller.cs,231,A,,,unread @@ -1861,12 +1861,12 @@ src/System.Management.Automation/engine/Interop/Windows/VariantClear.cs,15,B,,,u src/System.Management.Automation/engine/Interop/Windows/WNetAddConnection2.cs,73,B,,,unread src/System.Management.Automation/engine/Interop/Windows/WNetCancelConnection2.cs,37,B,,,unread src/System.Management.Automation/engine/Interop/Windows/WNetGetConnection.cs,86,B,,,unread -src/System.Management.Automation/engine/Modules/AnalysisCache.cs,1142,B,,,unread +src/System.Management.Automation/engine/Modules/AnalysisCache.cs,1142,B,44-140;645-690,RT-MODULES-001,partial src/System.Management.Automation/engine/Modules/ExportModuleMemberCommand.cs,192,B,,,unread src/System.Management.Automation/engine/Modules/GetModuleCommand.cs,610,B,,,unread src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs,2184,B,,,unread -src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs,7495,B,,,unread -src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs,1782,B,,,unread +src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs,7495,B,251-285;332-478;5422-5520;7022-7252;7375-7458,RT-MODULES-001,partial +src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs,1782,B,896-905;965-1031;1397-1700,RT-MODULES-001,partial src/System.Management.Automation/engine/Modules/ModuleSpecification.cs,384,B,,,unread src/System.Management.Automation/engine/Modules/ModuleUtils.cs,661,B,,,unread src/System.Management.Automation/engine/Modules/NewModuleCommand.cs,259,B,,,unread @@ -1888,12 +1888,12 @@ src/System.Management.Automation/engine/Subsystem/PredictionSubsystem/CommandPre src/System.Management.Automation/engine/Subsystem/PredictionSubsystem/ICommandPredictor.cs,297,B,,,unread src/System.Management.Automation/engine/WinRT/IInspectable.cs,41,B,,,unread src/System.Management.Automation/engine/debugger/Breakpoint.cs,612,B,,,unread -src/System.Management.Automation/engine/debugger/debugger.cs,5954,B,,,unread +src/System.Management.Automation/engine/debugger/debugger.cs,5954,B,31-60;1389-1440;1537-1604;5085-5160,RT-DEBUGGER-001,partial src/System.Management.Automation/engine/hostifaces/AsyncResult.cs,253,B,,,unread src/System.Management.Automation/engine/hostifaces/ChoiceDescription.cs,136,B,,,unread src/System.Management.Automation/engine/hostifaces/Command.cs,902,B,,,unread -src/System.Management.Automation/engine/hostifaces/Connection.cs,1895,B,,,unread -src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs,1611,B,,,unread +src/System.Management.Automation/engine/hostifaces/Connection.cs,1895,B,160-230;358-390,RT-PIPELINE-001,partial +src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs,1611,B,255-330;356-440;725-800;1020-1230,RT-PIPELINE-001,partial src/System.Management.Automation/engine/hostifaces/ConnectionFactory.cs,628,B,,,unread src/System.Management.Automation/engine/hostifaces/DefaultHost.cs,135,B,,,unread src/System.Management.Automation/engine/hostifaces/FieldDescription.cs,416,B,,,unread @@ -1904,23 +1904,23 @@ src/System.Management.Automation/engine/hostifaces/InternalHost.cs,581,B,,,unrea src/System.Management.Automation/engine/hostifaces/InternalHostRawUserInterface.cs,615,B,,,unread src/System.Management.Automation/engine/hostifaces/InternalHostUserInterface.cs,1033,B,,,unread src/System.Management.Automation/engine/hostifaces/ListModifier.cs,300,B,,,unread -src/System.Management.Automation/engine/hostifaces/LocalConnection.cs,1569,B,,,unread -src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs,1400,B,,,unread +src/System.Management.Automation/engine/hostifaces/LocalConnection.cs,1569,B,128-175;591-730,RT-PIPELINE-001,partial +src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs,1400,B,21-1147;1266-1400,RT-PIPELINE-001,partial src/System.Management.Automation/engine/hostifaces/MshHost.cs,328,B,,,unread src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs,1915,B,,,unread -src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs,1480,B,,,unread +src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs,1480,B,418-470;1190-1260,RT-ERRORS-001,partial src/System.Management.Automation/engine/hostifaces/PSCommand.cs,517,B,,,unread src/System.Management.Automation/engine/hostifaces/PSDataCollection.cs,2121,B,,,unread -src/System.Management.Automation/engine/hostifaces/PSTask.cs,1593,B,,,unread +src/System.Management.Automation/engine/hostifaces/PSTask.cs,1593,B,23-140;321-500;640-800,RT-JOBS-001,partial src/System.Management.Automation/engine/hostifaces/Parameter.cs,288,B,,,unread src/System.Management.Automation/engine/hostifaces/Pipeline.cs,740,B,,,unread -src/System.Management.Automation/engine/hostifaces/PowerShell.cs,6209,B,,,unread +src/System.Management.Automation/engine/hostifaces/PowerShell.cs,6209,B,100-200;4520-4600;5078-5190,RT-PIPELINE-001,partial src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs,293,B,,,unread src/System.Management.Automation/engine/hostifaces/RunspaceInit.cs,28,B,,,unread src/System.Management.Automation/engine/hostifaces/RunspacePool.cs,1388,B,,,unread -src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs,1669,B,,,unread +src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs,1669,B,718-806;1230-1300,RT-PIPELINE-001,partial src/System.Management.Automation/engine/hostifaces/internalHostuserInterfacesecurity.cs,74,B,,,unread -src/System.Management.Automation/engine/hostifaces/pipelinebase.cs,1072,B,,,unread +src/System.Management.Automation/engine/hostifaces/pipelinebase.cs,1072,B,266-360;425-570;580-700;790-900,RT-PIPELINE-001,partial src/System.Management.Automation/engine/interpreter/AddInstruction.cs,285,A,1-285,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read src/System.Management.Automation/engine/interpreter/ArrayOperations.cs,133,A,1-133,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read src/System.Management.Automation/engine/interpreter/BranchLabel.cs,129,A,1-129,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read @@ -1985,9 +1985,9 @@ src/System.Management.Automation/engine/parser/token.cs,1575,A,1-1575,LANG-TOKCA src/System.Management.Automation/engine/parser/tokenizer.cs,5083,A,1-5083,LANG-TOKCAT-001;LANG-TOKENIZER-001,read src/System.Management.Automation/engine/remoting/client/ClientMethodExecutor.cs,243,B,,,unread src/System.Management.Automation/engine/remoting/client/ClientRemotePowerShell.cs,1000,B,,,unread -src/System.Management.Automation/engine/remoting/client/Job.cs,4524,B,,,unread +src/System.Management.Automation/engine/remoting/client/Job.cs,4524,B,408-470;1386-1462,RT-JOBS-001,partial src/System.Management.Automation/engine/remoting/client/Job2.cs,2186,B,,,unread -src/System.Management.Automation/engine/remoting/client/JobManager.cs,1055,B,,,unread +src/System.Management.Automation/engine/remoting/client/JobManager.cs,1055,B,24-80,RT-JOBS-001,partial src/System.Management.Automation/engine/remoting/client/JobSourceAdapter.cs,545,B,,,unread src/System.Management.Automation/engine/remoting/client/PowerShellStreams.cs,259,B,,,unread src/System.Management.Automation/engine/remoting/client/RemoteRunspacePoolInternal.cs,2163,B,,,unread @@ -2094,13 +2094,13 @@ src/System.Management.Automation/help/AliasHelpProvider.cs,322,B,,,unread src/System.Management.Automation/help/BaseCommandHelpInfo.cs,489,B,,,unread src/System.Management.Automation/help/CabinetAPI.cs,139,B,,,unread src/System.Management.Automation/help/CabinetNativeApi.cs,822,B,,,unread -src/System.Management.Automation/help/CommandHelpProvider.cs,1413,B,,,unread +src/System.Management.Automation/help/CommandHelpProvider.cs,1413,B,189-300;425-640,RT-HELP-001,partial src/System.Management.Automation/help/DefaultCommandHelpObjectBuilder.cs,934,B,,,unread src/System.Management.Automation/help/DefaultHelpProvider.cs,65,B,,,unread src/System.Management.Automation/help/DscResourceHelpProvider.cs,371,B,,,unread src/System.Management.Automation/help/HelpCategoryInvalidException.cs,126,B,,,unread src/System.Management.Automation/help/HelpCommands.cs,904,B,,,unread -src/System.Management.Automation/help/HelpCommentsParser.cs,1226,B,,,unread +src/System.Management.Automation/help/HelpCommentsParser.cs,1226,B,712-810;863-915,RT-HELP-001,partial src/System.Management.Automation/help/HelpErrorTracer.cs,207,B,,,unread src/System.Management.Automation/help/HelpFileHelpInfo.cs,139,B,,,unread src/System.Management.Automation/help/HelpFileHelpProvider.cs,417,B,,,unread @@ -2110,7 +2110,7 @@ src/System.Management.Automation/help/HelpProvider.cs,255,B,,,unread src/System.Management.Automation/help/HelpProviderWithCache.cs,231,B,,,unread src/System.Management.Automation/help/HelpProviderWithFullCache.cs,100,B,,,unread src/System.Management.Automation/help/HelpRequest.cs,178,B,,,unread -src/System.Management.Automation/help/HelpSystem.cs,926,B,,,unread +src/System.Management.Automation/help/HelpSystem.cs,926,B,158-260;352-460;563-640,RT-HELP-001,partial src/System.Management.Automation/help/HelpUtils.cs,69,B,,,unread src/System.Management.Automation/help/MUIFileSearcher.cs,341,B,,,unread src/System.Management.Automation/help/MamlClassHelpInfo.cs,134,B,,,unread @@ -2150,8 +2150,8 @@ src/System.Management.Automation/namespaces/IContentWriter.cs,69,B,,,unread src/System.Management.Automation/namespaces/IDynamicPropertyProvider.cs,339,B,,,unread src/System.Management.Automation/namespaces/IPermissionProvider.cs,110,B,,,unread src/System.Management.Automation/namespaces/IPropertiesProvider.cs,201,B,,,unread -src/System.Management.Automation/namespaces/ItemProviderBase.cs,690,B,,,unread -src/System.Management.Automation/namespaces/LocationGlobber.cs,4732,B,,,unread +src/System.Management.Automation/namespaces/ItemProviderBase.cs,690,B,22-330,RT-PROVIDER-001,partial +src/System.Management.Automation/namespaces/LocationGlobber.cs,4732,B,732-790;1348-1445;1469-1560;2380-2454,RT-PROVIDER-001,partial src/System.Management.Automation/namespaces/NavigationProviderBase.cs,1091,B,,,unread src/System.Management.Automation/namespaces/PathInfo.cs,165,B,,,unread src/System.Management.Automation/namespaces/ProviderBase.cs,1986,B,,,unread diff --git a/docs-out/.work/ledger-proposals/T31_T39.csv b/docs-out/.work/ledger-proposals/T31_T39.csv new file mode 100644 index 0000000..82a7c4f --- /dev/null +++ b/docs-out/.work/ledger-proposals/T31_T39.csv @@ -0,0 +1,57 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/Pipe.cs,1-622,RT-PIPELINE-001 +src/System.Management.Automation/engine/pipeline.cs,1-1668,RT-PIPELINE-001;RT-ERRORS-001 +src/System.Management.Automation/engine/CommandProcessorBase.cs,20-1044,RT-PIPELINE-001;RT-CMDLET-001 +src/System.Management.Automation/engine/CommandProcessor.cs,1-869,RT-PIPELINE-001;RT-BIND-001 +src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs,21-1147;1266-1400,RT-PIPELINE-001 +src/System.Management.Automation/engine/hostifaces/pipelinebase.cs,266-360;425-570;580-700;790-900,RT-PIPELINE-001 +src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs,255-330;356-440;725-800;1020-1230,RT-PIPELINE-001 +src/System.Management.Automation/engine/hostifaces/LocalConnection.cs,128-175;591-730,RT-PIPELINE-001 +src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs,718-806;1230-1300,RT-PIPELINE-001 +src/System.Management.Automation/engine/hostifaces/PowerShell.cs,100-200;4520-4600;5078-5190,RT-PIPELINE-001 +src/System.Management.Automation/engine/hostifaces/Connection.cs,160-230;358-390,RT-PIPELINE-001 +src/System.Management.Automation/engine/AutomationEngine.cs,56-129,RT-PIPELINE-001 +src/System.Management.Automation/engine/CommandSearcher.cs,22-1100;1322-1640,RT-DISCOVERY-001 +src/System.Management.Automation/engine/CommandDiscovery.cs,276-660;862-1260;1284-1464,RT-DISCOVERY-001 +src/System.Management.Automation/engine/NativeCommandProcessor.cs,197-232;557-606;715-967;1082-1240;1449-1514;1586-1884,RT-DISCOVERY-001 +src/System.Management.Automation/engine/NativeCommandParameterBinder.cs,58-470,RT-DISCOVERY-001 +src/System.Management.Automation/engine/InitialSessionState.cs,4655-4685;5515-5535,RT-DISCOVERY-001;RT-PROVIDER-001 +src/System.Management.Automation/engine/CommandInfo.cs,20-80,RT-DISCOVERY-001 +src/System.Management.Automation/engine/ParameterBinderController.cs,133-330;514-870;919-1190,RT-BIND-001 +src/System.Management.Automation/engine/CmdletParameterBinderController.cs,104-470;854-990;1198-1320;1463-1580;1595-1735;1756-2000;2532-2700;2752-2905;3147-3640,RT-BIND-001 +src/System.Management.Automation/engine/ParameterBinderBase.cs,329-667,RT-BIND-001 +src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs,78-160;164-176;278-345;445-580,RT-BIND-001 +src/System.Management.Automation/engine/cmdlet.cs,120-215;1738-1770,RT-CMDLET-001 +src/System.Management.Automation/engine/CommandBase.cs,250-270;300-400,RT-CMDLET-001;RT-ERRORS-001 +src/System.Management.Automation/engine/MshCommandRuntime.cs,164-292;667-742;1446-1655;2325-2400;2648-2730;2858-3010;3516-3600,RT-CMDLET-001;RT-ERRORS-001 +src/System.Management.Automation/namespaces/LocationGlobber.cs,732-790;1348-1445;1469-1560;2380-2454,RT-PROVIDER-001 +src/System.Management.Automation/namespaces/ItemProviderBase.cs,22-330,RT-PROVIDER-001 +src/System.Management.Automation/engine/SessionStateProviderAPIs.cs,253-297;953-1070,RT-PROVIDER-001 +src/System.Management.Automation/engine/SessionStateDriveAPIs.cs,64-232;410-477,RT-PROVIDER-001 +src/System.Management.Automation/engine/DataStoreAdapterProvider.cs,546-638,RT-PROVIDER-001 +src/System.Management.Automation/engine/MshObject.cs,245-330;426-545;1938-2116,RT-ETS-001 +src/System.Management.Automation/engine/MshMemberInfo.cs,33-150;4503-4560;4799-4860,RT-ETS-001 +src/System.Management.Automation/engine/TypeTable.cs,3965-4110,RT-ETS-001 +src/System.Management.Automation/engine/CoreAdapter.cs,38-330;2616-2700,RT-ETS-001 +src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs,36-170,RT-FORMAT-001 +src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs,1-400,RT-FORMAT-001 +src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs,19-326,RT-FORMAT-001 +src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs,16-166;256-400,RT-FORMAT-001 +src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs,91-215;399-467,RT-FORMAT-001 +src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs,333-586,RT-FORMAT-001 +src/System.Management.Automation/FormatAndOutput/format-default/format-default.cs,1-39,RT-FORMAT-001 +src/System.Management.Automation/help/HelpSystem.cs,158-260;352-460;563-640,RT-HELP-001 +src/System.Management.Automation/help/CommandHelpProvider.cs,189-300;425-640,RT-HELP-001 +src/System.Management.Automation/help/HelpCommentsParser.cs,712-810;863-915,RT-HELP-001 +src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs,251-285;332-478;5422-5520;7022-7252;7375-7458,RT-MODULES-001 +src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs,896-905;965-1031;1397-1530;1529-1700,RT-MODULES-001 +src/System.Management.Automation/engine/Modules/AnalysisCache.cs,44-140;645-690,RT-MODULES-001 +src/System.Management.Automation/engine/ExecutionContext.cs,818-858,RT-ERRORS-001 +src/System.Management.Automation/engine/QuestionMarkVariable.cs,1-45,RT-ERRORS-001 +src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs,418-470;1190-1260,RT-ERRORS-001 +src/System.Management.Automation/engine/remoting/client/Job.cs,408-470;1386-1462,RT-JOBS-001 +src/System.Management.Automation/engine/remoting/client/JobManager.cs,24-80,RT-JOBS-001 +src/System.Management.Automation/engine/hostifaces/PSTask.cs,23-140;321-500;640-800,RT-JOBS-001 +src/System.Management.Automation/engine/InternalCommands.cs,240-271;377-470,RT-JOBS-001 +src/System.Management.Automation/engine/debugger/debugger.cs,31-60;1389-1440;1537-1604;5085-5160,RT-DEBUGGER-001 +src/System.Management.Automation/engine/EventManager.cs,22-140;326-392;510-572;645-796;957-1137;1222-1295;1821-1850,RT-EVENTING-001 diff --git a/docs-out/02-language/RUNTIME-OPERATIONS.md b/docs-out/02-language/RUNTIME-OPERATIONS.md index 8a10e6e..4ed2b59 100644 --- a/docs-out/02-language/RUNTIME-OPERATIONS.md +++ b/docs-out/02-language/RUNTIME-OPERATIONS.md @@ -119,7 +119,7 @@ Called from the per-statement catch block of compiled code ([02.5-compiler-and-e 6. Else if the preference is `Inquire` (and prompting is allowed) → ask the host. 7. `SilentlyContinue`/`Ignore` → return (execution resumes at the next statement). `Stop` → mark prompts suppressed and rethrow. Otherwise (`Continue`): if there is no trap and the exception came from a `throw` statement → rethrow; else report the error record to the error stream (`ReportErrorRecord`) and, if it cannot be reported, rethrow. -This is the rule that makes ordinary errors in script statements non-terminating by default while `throw` and `-ErrorAction Stop` terminate. The terminating/non-terminating canonical fact CF-ERR-001 is owned by the errors document (planned T38). +This is the rule that makes ordinary errors in script statements non-terminating by default while `throw` and `-ErrorAction Stop` terminate. The terminating/non-terminating canonical fact CF-ERR-001 is owned by [errors-and-diagnostics](../03-runtime/errors-and-diagnostics.md). ### 6.2 Collections and arrays diff --git a/docs-out/03-runtime/cmdlet-infrastructure.md b/docs-out/03-runtime/cmdlet-infrastructure.md new file mode 100644 index 0000000..56ce1f4 --- /dev/null +++ b/docs-out/03-runtime/cmdlet-infrastructure.md @@ -0,0 +1,174 @@ +--- +id: RT-CMDLET-001 +title: Cmdlet infrastructure +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/cmdlet.cs, src/System.Management.Automation/engine/CommandBase.cs, src/System.Management.Automation/engine/MshCmdlet.cs, src/System.Management.Automation/engine/MshCommandRuntime.cs, src/System.Management.Automation/engine/CommonCommandParameters.cs, src/System.Management.Automation/engine/UserFeedbackParameters.cs, src/System.Management.Automation/engine/Attributes.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/types.json, data/attributes.json, data/tests.json, data/enforcement-points.json] +--- + +# Cmdlet infrastructure + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Lifecycle ordering is CF-LCY-001 in [runspace-and-pipeline](runspace-and-pipeline.md); binding is in [parameter-binding](parameter-binding.md); script cmdlets (advanced functions) in [SCRIPT-BLOCKS](../02-language/SCRIPT-BLOCKS.md); error semantics in [errors-and-diagnostics](errors-and-diagnostics.md). + +## 1. Purpose and scope [SPEC] + +A *cmdlet* is a command implemented as a .NET class. The base classes (`InternalCommand` → `Cmdlet` → `PSCmdlet`) define the lifecycle hooks and the API a cmdlet uses to write output, errors and other streams, ask for confirmation (`ShouldProcess`, `ShouldContinue`), and reach the engine. The *command runtime* (`MshCommandRuntime`) is the engine-side object behind those calls: it applies stream preferences, merges streams, writes to pipes and enforces when writing is allowed. Advanced functions use the same runtime through `PSScriptCmdlet`. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/CommandBase.cs::InternalCommand@L39-L283 | abstract class | internal base: lifecycle `Do*` hooks, `MyInvocation`, `Stopping`, `CurrentPipelineObject`, `Context` | B | +| src/System.Management.Automation/engine/cmdlet.cs::Cmdlet@L33-L1829 | abstract class | public base: `BeginProcessing`…`StopProcessing`, `Write*`, `ShouldProcess`, `ShouldContinue`, `ThrowTerminatingError` | B | +| src/System.Management.Automation/engine/CommandBase.cs::PSCmdlet@L415-L603 | abstract partial class | adds `SessionState`, `Host`, `Events`, `InvokeProvider`, `InvokeCommand`, `ParameterSetName`, path helpers, variable access | B | +| src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime@L29-L3829 | class | the command runtime: pipes, preferences, write permission, `ShouldProcess`, error-action | B | +| src/System.Management.Automation/engine/DefaultCommandRuntime.cs::DefaultCommandRuntime@L15-L245 | class | a runtime for cmdlets invoked directly as .NET objects (no pipeline) | B | +| src/System.Management.Automation/engine/CommonCommandParameters.cs::CommonParameters@L13-L283 | sealed class | the common parameters | B | +| src/System.Management.Automation/engine/UserFeedbackParameters.cs::ShouldProcessParameters@L104-L166 | sealed class | `-WhatIf`, `-Confirm` | B | +| src/System.Management.Automation/engine/UserFeedbackParameters.cs::PagingParameters@L13-L96 | sealed class | `-First`, `-Skip`, `-IncludeTotalCount` | B | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552 | sealed class | the cmdlet that runs an advanced function's script block | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the hook contract (what a cmdlet may override and when it is called), the stream-writing API semantics, preference-driven behavior of verbose/debug/warning/information/progress/error, confirmation logic, write-permission windows. Does not own: pipeline ordering (T31), parameter binding (T33), formatting of what is written (T36), the cmdlet inventory (Phase 4 module families). + +## 4. Key types and contracts [IMPL] + +- **Lifecycle hooks**: `Cmdlet.BeginProcessing`, `ProcessRecord`, `EndProcessing`, `StopProcessing` are `protected virtual` and empty; the engine calls internal `DoBeginProcessing` / `DoProcessRecord` / `DoEndProcessing` / `DoStopProcessing`, which for a cmdlet forward to them; `DoBeginProcessing` first verifies that a transaction exists when `-UseTransaction` was given (`InvalidOperationException` with `NoTransactionStarted*` text otherwise) (src/System.Management.Automation/engine/cmdlet.cs::Cmdlet.DoBeginProcessing@L141-L166). `DoCleanResource` is the hook of the `clean` phase (script cmdlets). +- **`Stopping`** is true once the pipeline is stopping; the runtime calls `ThrowIfStopping` before every write, which throws `PipelineStoppedException`. +- **Write API** (`WriteObject(obj)`, `WriteObject(obj, enumerateCollection)`, `WriteError`, `WriteWarning`, `WriteVerbose`, `WriteDebug`, `WriteInformation`, `WriteProgress`, `WriteCommandDetail`) forwards to the command runtime. `WriteObject` checks stopping, checks that writing is permitted, drops `AutomationNull`, wraps the value as `PSObject` (`AsPSObjectOrNull`) and adds it to the output pipe; with `enumerateCollection` an enumerable is expanded and its non-`AutomationNull` elements added together via `AddItems` (src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteObjectSkipAllowCheck@L2648-L2658, src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._EnumerateAndWriteObjectSkipAllowCheck@L2672-L2698). +- **Write permission**: writing is allowed only while the pipeline processor's `_permittedToWrite` equals this command, on the thread that was recorded when the permission was granted, and (for pipeline writes) when `_permittedToWriteToPipeline` is true; otherwise `WriteNotPermitted`. The permission is granted by `AllowThisCommandToWrite` (a disposable that saves and restores the three fields) around binding (no pipeline writes), begin, process, end and clean (src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.ThrowIfWriteNotPermitted@L2325-L2341, src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.AllowThisCommandToWrite@L2349-L2352). This means a cmdlet that stores a runtime reference and writes from another thread fails. +- **Error stream** (`_WriteErrorSkipAllowCheck`): marks the pipeline `ExecutionFailed`; picks the action preference (explicit argument or `ErrorAction`); `Ignore` returns without writing or recording; `SilentlyContinue` appends to the error variables and returns; if a previous prompt answered "yes to all" the preference becomes `Continue`; `Stop` throws `ActionPreferenceStopException` (via `ManageException`); `Inquire` asks the host; otherwise the record is appended to the error variables and written, as a `PSObject` marked with the error stream, to the output pipe when the error stream is merged into output (`2>&1`), else to the error pipe (transcribed first if transcribing). Native-stderr text uses the same path without setting failure (src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968). +- **Other streams** use one rule (`WriteHelper`, `WriteWarning`): if the preference is `Ignore`, `SilentlyContinue`, `Continue` or `Break` the answer is "yes"; `Stop` throws `ActionPreferenceStopException` with the preference variable name; `Inquire` prompts (allowing "yes to all"); remembered "yes/no to all" answers short-circuit later calls. A warning is written to the warning pipe (as `PSObject` with the warning stream tag) or, without a pipe, to the host; a `Break` preference enters the debugger; the record is always appended to the warning variables (src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.WriteWarning@L678-L738, src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.WriteHelper@L3522-L3573). +- **`ShouldProcess`** (`DoShouldProcess`): a remembered "no to all"/"yes to all" answer wins; with `-WhatIf` the message "What if: …" is written to the host and the call returns false; if the command's `ConfirmImpact` is below the `-Confirm`/`$ConfirmPreference` threshold (or the threshold is `None`) the call returns true, writing the description to the verbose stream when `-Verbose`; otherwise the user is asked (yes, no, yes-to-all, no-to-all, halt) and the answer decides (src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.DoShouldProcess@L1515-L1621). `-Confirm` forces the threshold to `Low`; an explicit `-Confirm:$false` sets it to `None`. A runtime "optimization" query reports whether the call can be skipped (auto-yes without verbose), made asynchronously (auto-yes with verbose, or what-if) or must be synchronous. +- **Common parameters**: `-Verbose` (`vb`), `-Debug` (`db`), `-ErrorAction` (`ea`), `-WarningAction` (`wa`), `-InformationAction` (`infa`), `-ProgressAction` (`proga`), `-ErrorVariable` (`ev`), `-WarningVariable` (`wv`), `-InformationVariable` (`iv`), `-OutVariable` (`ov`), `-OutBuffer` (`ob`), `-PipelineVariable` (`pv`) (src/System.Management.Automation/engine/CommonCommandParameters.cs::CommonParameters@L13-L283). They are forwarded to the command runtime. `-OutBuffer` sets the output pipe's `OutBufferCount` (see [runspace-and-pipeline](runspace-and-pipeline.md)). Variable parameters are set up by the pipeline processor after binding (`SetupOutVariable` etc.). + +## 5. Data model + +`MshCommandRuntime` holds: the three standard pipes (`InputPipe`, `OutputPipe`, `ErrorOutputPipe`) plus optional warning/verbose/debug/information/progress pipes for redirection, the `PipelineProcessor` back-reference, per-stream "last continue status" (for yes/no-to-all), cached preference values, merge settings (`ErrorMergeTo`, etc.), `MergeUnclaimedPreviousErrorResults`, `IsClosed` (no more input), and the variable lists for `-OutVariable` and friends. `InternalCommand` holds `MyInvocation` (`InvocationInfo`: name, position message, pipeline position and length, bound parameters, unbound arguments, `CommandOrigin`, history id) and `CurrentPipelineObject` (`$_`/`$PSItem`). + +## 6. Control flow + +```mermaid +sequenceDiagram + participant CP as CommandProcessor + participant C as Cmdlet + participant R as MshCommandRuntime + participant P as Pipe + CP->>R: AllowThisCommandToWrite(true) + CP->>C: DoProcessRecord + C->>C: ProcessRecord (user code) + C->>R: WriteObject(x) + R->>R: ThrowIfStopping, ThrowIfWriteNotPermitted + R->>P: OutputPipe.Add (AsPSObjectOrNull) + C->>R: WriteError(record) + R->>R: ErrorAction decision (Ignore, SilentlyContinue, Stop, Inquire, Continue) + R->>P: ErrorOutputPipe or OutputPipe when merged +``` + +Ordered steps for one `WriteError`: stopping check → text-lookup error logged → execution-detail log → mark failure → preference (Ignore → silent return; SilentlyContinue → record in variables, return; Stop → throw; Inquire → prompt) → record in variables → wrap and write to the error or merged-output pipe. + +## 7. Algorithms + +Stream decisions are table-driven by `ActionPreference` (`SilentlyContinue`, `Stop`, `Continue`, `Inquire`, `Ignore`, `Suspend` (not seen here), `Break`); the table for `WriteHelper` is in §4. `ShouldProcess` is the decision list in §4. No further non-trivial algorithm in this document (error-action propagation across statements is in [errors-and-diagnostics](errors-and-diagnostics.md)). + +## 8. Observable behavior [SPEC] + +- A cmdlet that writes outside its lifecycle windows or from a foreign thread fails with "the command cannot write at this time". +- `-ErrorAction Ignore` leaves no trace in `$Error`; `SilentlyContinue` records but prints nothing; `Stop` makes the error terminating (`ActionPreferenceStopException`); `-ErrorVariable` receives records even when silenced. +- `-WhatIf` prints "What if: …" and the cmdlet skips the action; `-Confirm` prompts for commands whose impact is at or above the preference. +- `$_` inside `process` is the current pipeline object. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`ThrowTerminatingError` goes through the runtime (`ManageException`) when a runtime exists, else rethrows the record's exception or an `InvalidOperationException`; it runs inside an engine protection scope (transactions). Writing after the pipeline started stopping throws `PipelineStoppedException`. A host-less runtime for warnings asserts. A transaction is required only if `-UseTransaction` was bound. + +## 10. Concurrency and threading + +Cmdlet code runs on the pipeline thread; the write-permission record includes the thread, so asynchronous callbacks must marshal back to it (the documented pattern is to collect results and write from `ProcessRecord`/`EndProcessing`). `StopProcessing` is called from another thread (see CF-LCY-001), so cmdlets must make it thread-safe. `DefaultCommandRuntime` exists for direct .NET invocation without a pipeline. + +## 11. Extension points and customization hooks + +Derive from `Cmdlet` or `PSCmdlet` and add `[Cmdlet(verb, noun)]` (`CmdletAttribute`: `DefaultParameterSetName`, `SupportsShouldProcess`, `SupportsPaging`, `SupportsTransactions`, `ConfirmImpact`, `RemotingCapability`), `[OutputType]`, parameters with `[Parameter]`, `[Alias]`, validation attributes (`ValidateSet`, `ValidateRange`, `ValidatePattern`, `ValidateLength`, `ValidateCount`, `ValidateScript`, `ValidateNotNull*`, `ValidateTrustedData`, `ValidateUserDrive`, `AllowNull`, `AllowEmptyString`, `AllowEmptyCollection`, argument transformation) (src/System.Management.Automation/engine/Attributes.cs). Dynamic parameters via `IDynamicParameters`. Advanced functions use `[CmdletBinding()]` and the `begin`/`process`/`end`/`clean` blocks; `PSScriptCmdlet` adapts them to the same hooks and exposes `$PSCmdlet`. + +## 12. Dependencies + +Inbound: `CommandProcessor`, the compiler (for `$PSCmdlet` methods), module authors. Outbound: `PipelineProcessor` (write-permission fields, `ExecutionFailed`), `Pipe`, the host UI (prompts, verbose/debug/warning display), `InternalHost`, `PSTransactionManager`, the debugger (`Break`), `MshLog`. + +## 13. Performance notes + +`ShouldProcess` has an optimization query so callers can avoid building descriptions when the answer is predictable. Preferences are cached per command (`_isConfirmPreferenceCached`). Every `WriteObject` wraps in `PSObject` (allocation) and re-checks stopping and permission; the compiler special-cases some operations to avoid it (not read here). + +## 14. Security considerations + +Enforced here: write-permission windows (a cmdlet cannot inject output into another command's pipe at an arbitrary time or thread); `ConfirmImpact` and `-Confirm`/`-WhatIf` as a user-consent mechanism; the debugger break on warnings/errors with `Break`; transcript hooks on error and warning output. Not enforced here: what a cmdlet is allowed to do (cmdlets are trusted .NET code). The `ValidateTrustedData` attribute marks parameters that must not receive untrusted data in constrained language (`UNVERIFIED` enforcement location, Phase 5). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 4 | +| **total** | 4 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/MshCommandRuntime.cs` | 4 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: a command can write only through its own runtime and only inside its windows; consent (`ShouldProcess`) results are recorded in audit logs with the decision source (what-if, preference, user); trusted-data validation is part of the parameter contract. + +## 15. Enterprise considerations + +Security first: §14. Observability: error/warning/verbose/debug/information/progress are separate typed streams with records that carry the invocation info; they are transcribed and can be captured in variables. Supportability: `ErrorRecord` carries category, target and recommended action. Testability: `DefaultCommandRuntime` lets a cmdlet run without an engine pipeline. Resource limits: none on stream volume. Missing: structured, schema-versioned output records beyond `PSObject`. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Scripting/CommonParameters.Tests.ps1` | 1 | 10 | 22 | +| `test/powershell/Language/Scripting/Dynamicparameters.Tests.ps1` | 1 | 0 | 5 | +| `test/powershell/engine/Basic/DefaultCommands.Tests.ps1` | 2 | 0 | 6 | +| **total (3 files)** | 4 | 10 | 33 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `Scripting/CmdletBinding`, `Scripting/ShouldProcess`, `engine/Basic/DefaultCommands`, `CommonParameters`, `Scripting/AdvancedFunction`, `Scripting/Functions`, `Scripting/Dynamicparameters`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: all stream behavior sits in one runtime class, so cmdlets and script commands share behavior; the price is a 3,800-line class. +- `INFERENCE`: the thread-and-window write check exists to keep the single-threaded pipeline model sound (a cmdlet writing from a timer thread would corrupt downstream state). +- `FACT-SRC`: preferences are variables read through normal scope lookup (see [execution-context-and-session-state](execution-context-and-session-state.md)), so scripts can set them per scope. +- `FACT-SRC`: "yes to all" memory is kept per stream per command instance. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTCMD-01** [REPLICATE] (area: NLD-CMD) Keep the four hooks (begin, process, end, stop) plus clean, with `Stopping` checked on every write. Evidence: src/System.Management.Automation/engine/cmdlet.cs::Cmdlet.DoBeginProcessing@L141-L166. +- **LSN-RTCMD-02** [REPLICATE] (area: NLD-CMD) Separate typed streams with per-stream action preferences and the Ignore / SilentlyContinue / Continue / Inquire / Stop semantics. Evidence: src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968. +- **LSN-RTCMD-03** [REPLICATE] (area: NLD-CMD) `ShouldProcess` decision order (remembered answer → what-if → impact threshold → prompt). Evidence: src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.DoShouldProcess@L1515-L1621. +- **LSN-RTCMD-04** [ADAPT] (area: NLD-SEC) Make write permission a capability object passed to the cmdlet for the duration of a phase instead of three mutable fields on the pipeline processor checked against the current thread. Evidence: src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.AllowThisCommandToWrite@L2349-L2352. +- **LSN-RTCMD-05** [ADAPT] (area: NLD-CMD) Split the 3,800-line runtime into stream writers (one per stream) behind one interface, keeping behavior identical. Evidence: src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime@L29-L3829. +- **LSN-RTCMD-06** [AVOID] (area: NLD-CMD) Do not carry transaction support (`-UseTransaction`, `SupportsTransactions`, the engine protection scope) into the new shell; the feature is Windows-only and out of core scope. DEV: DEV-CMDLET-135. Evidence: src/System.Management.Automation/engine/cmdlet.cs::Cmdlet.DoBeginProcessing@L141-L166. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-033): the rest of `MshCommandRuntime` (verbose/debug/information/progress writers, `ManageException`, `InquireHelper` prompt rules, variable setup and `AppendErrorToVariables`, `IsPipelineInputExpected`), `Cmdlet` members not listed, `PSCmdlet` helper members, `PSScriptCmdlet` internals (covered functionally in [SCRIPT-BLOCKS](../02-language/SCRIPT-BLOCKS.md)), `DefaultCommandRuntime`. +- `UNVERIFIED` (VB-034): `CmdletInfo`/`CmdletAttribute` metadata flow into `CommandMetadata`. + +Verification: `cmdlet.cs` lines 120–215 and 1738–1770, `CommandBase.cs` lines 250–270, `MshCommandRuntime.cs` lines 164–292, 667–742, 1446–1655, 2325–2400, 2648–2730, 2858–3010, 3516–3600, and the first lines of `CommonCommandParameters.cs` (aliases) were read; ranges are in `T31_T39.csv`. This document is tier B: contracts and the algorithms above were read, other members were taken from signatures only. diff --git a/docs-out/03-runtime/command-discovery-and-invocation.md b/docs-out/03-runtime/command-discovery-and-invocation.md new file mode 100644 index 0000000..7ccaf3f --- /dev/null +++ b/docs-out/03-runtime/command-discovery-and-invocation.md @@ -0,0 +1,207 @@ +--- +id: RT-DISCOVERY-001 +title: Command discovery and invocation +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-DSC-001] +sources: [src/System.Management.Automation/engine/CommandDiscovery.cs, src/System.Management.Automation/engine/CommandSearcher.cs, src/System.Management.Automation/engine/CommandInfo.cs, src/System.Management.Automation/engine/NativeCommandProcessor.cs, src/System.Management.Automation/engine/NativeCommandParameterBinder.cs, src/System.Management.Automation/engine/NativeCommandParameterBinderController.cs, src/System.Management.Automation/engine/ScriptCommandProcessor.cs, src/System.Management.Automation/engine/CommandProcessor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/members-tierA.json, data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Command discovery and invocation + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. The pipeline that drives the processors is in [runspace-and-pipeline](runspace-and-pipeline.md); binding in [parameter-binding](parameter-binding.md); module auto-loading in [modules](modules.md); script execution inside a processor in [LIFECYCLE-script-execution](../02-language/LIFECYCLE-script-execution.md) and [SCRIPT-BLOCKS](../02-language/SCRIPT-BLOCKS.md). + +## 1. Purpose and scope [SPEC] + +Turn a command *name* typed in a script (or given to the API) into a *command processor* that can run. Two steps: **discovery** (name → `CommandInfo`: alias, function, filter, cmdlet, external script, application, script, configuration) and **invocation** (`CommandInfo` → processor: `CommandProcessor` for cmdlets and advanced functions, `DlrScriptCommandProcessor` for other script code, `NativeCommandProcessor` for external programs). Native process handling (argument passing, redirection, encoding, lifecycle) is part of this document. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery@L118-L1464 | class | name → info → processor; module auto-load; event hooks; PATH cache | A | +| src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher@L22-L1708 | class | the resolution state machine (an enumerator of matches) | A | +| src/System.Management.Automation/engine/CommandPathSearch.cs::CommandPathSearch@L18-L595 | class | enumerates files in the lookup directories (used by the searcher) | A | +| src/System.Management.Automation/engine/CommandInfo.cs::CommandInfo@L82-L737 | abstract class | base of `AliasInfo`, `FunctionInfo`, `FilterInfo`, `CmdletInfo`, `ExternalScriptInfo`, `ApplicationInfo`, `ScriptInfo`, `ConfigurationInfo` | A | +| src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor@L197-L1955 | class | runs an external program | A | +| src/System.Management.Automation/engine/NativeCommandParameterBinder.cs::NativeCommandParameterBinder@L21-L519 | class | turns bound arguments into a command line and an argument list | A | +| src/System.Management.Automation/engine/ScriptCommandProcessor.cs::DlrScriptCommandProcessor@L232-L661 | class | runs compiled script code as a command | A | + +## 3. Responsibilities and non-responsibilities + +Owns: the resolution order, the command-type set, visibility and authorization checks at lookup time, the lookup events (`PreCommandLookupAction`, `CommandNotFoundAction`, `PostCommandLookupAction`), requirement checks for script files (`#requires`), native process start/stop and stream wiring. Does not own: the contents of the cmdlet/function/alias tables (session state, T29/T37), module analysis (T37), parameter binding for cmdlets and scripts (T33), scope creation (T29). + +## 4. Key types and contracts [IMPL] + +- `CommandTypes` (flags): `Alias=0x1`, `Function=0x2`, `Filter=0x4`, `Cmdlet=0x8`, `ExternalScript=0x10`, `Application=0x20`, `Script=0x40`, `Configuration=0x100`, `All` = the union (src/System.Management.Automation/engine/CommandInfo.cs::CommandTypes@L23-L76). **There is no `Workflow` member in this version** (the study prompt lists one; `VERSION-MISMATCH`, recorded as a hint-vs-reality item). +- `CommandSearcher` implements `IEnumerable` over itself: each `MoveNext` yields the next match for one name in a fixed state order (§7). It is constructed with the name, `SearchResolutionOptions` (pattern flags, abbreviation expansion, search all scopes, `ResolveLiteralThenPathPatterns`), the allowed `CommandTypes`, and the context; `CommandOrigin` is set separately (src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239). +- `CommandDiscovery.LookupCommandInfo(name, types, options, origin, context)` is the static lookup entry; the instance overloads default to `CommandTypes.All`, `ResolveLiteralThenPathPatterns` and `CommandOrigin.Internal` (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.LookupCommandInfo@L641-L644). `LookupCommandProcessor(name | info, origin, useLocalScope, …)` returns a ready processor and sets `InvocationName` to the name used (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.LookupCommandProcessor@L276-L291). +- `CommandOrigin` has two values used here: `Runspace` (name typed by a user or the API) and `Internal` (name written in engine or module code). Visibility checks apply to `Runspace` origin only. +- Result of processor creation by `CommandInfo` type (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.LookupCommandProcessor@L276-L291): `Application` → `NativeCommandProcessor`; `Cmdlet` → `CommandProcessor`; `ExternalScript` → (requirement checks, then) script processor; `Filter`/`Function`/`Configuration` → script processor over the function's script block; `Script` → script processor; anything else (including an unresolved alias) → `CommandNotFoundException`. A script processor is a `CommandProcessor` (with a `PSScriptCmdlet`) when the script block uses `[CmdletBinding()]`, otherwise a `DlrScriptCommandProcessor` (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.GetScriptAsCmdletProcessor@L615-L625). Default for `useLocalScope` is **true** (a new scope per call). + +## 5. Data model + +`CommandInfo` carries: name, type, definition, module name and module, visibility (`SessionStateEntryVisibility`), the defining language mode, `CommandMetadata` (parameters), and for aliases the resolved target. Lookup tables live in session state: alias table, function table, cmdlet table (name → list of `CmdletInfo`, because several modules may export the same cmdlet name), plus allow-lists `Applications` and `Scripts` (strings) used in restricted sessions (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.GetCmdletInfo@L1358-L1423, src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239). The discovery object caches the split `PATH` (keyed by the raw `PATH` string, with `~` and `~/` expanded) and the lower-cased `PATHEXT` list (keyed by the raw value) (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.GetLookupDirectoryPaths@L1200-L1259, src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.InitPathExtCache@L1319-L1332). + +## 6. Control flow + +```mermaid +flowchart TD + N[command name] --> PRE{PreCommandLookupAction} + PRE -->|StopSearch with command| R[result] + PRE -->|continue| NS[TryNormalSearch via CommandSearcher] + NS -->|found| POST + NS -->|not found, no dash and no backslash| G[retry as Get-name] + G --> POST + NS -->|not found| AL[TryModuleAutoLoading: Module backslash Command] + AL -->|found| POST + AL -->|not found, preference All| AD[TryModuleAutoDiscovery: scan module analysis cache] + AD -->|found| POST + AD -->|not found| CNF[CommandNotFoundAction handler] + CNF -->|command set| POST + CNF -->|none| ERR[CommandNotFoundException] + POST{PostCommandLookupAction may replace} --> R + R --> LP[LookupCommandProcessor: resolve alias chain, ShouldRun, create processor] +``` + +**Command resolution order (CF-DSC-001).** For one name the searcher tries, in this order and returning the first match that is allowed: (1) **alias** (a match whose visibility does not allow the caller's origin is skipped and the search continues with functions); (2) **function / filter / configuration**; (3) **cmdlet** (a module-qualified name `Module\Cmdlet` matches only cmdlets of that module; with several cmdlets of the same name in different modules, the first in table order is returned unless all scopes are searched); (4) if application or external-script types are allowed: a name that is a *qualified provider path* is resolved as a path; then an absolute path to an existing file; then the file system search of each `PATH` directory in order (for a name with a directory part: only that directory); then a PowerShell-relative path (name starting with `.`, `~` or `\`). Within one directory the candidate file names are, in order: the name itself first if it already has an extension; `name.ps1` when external scripts are allowed (also `.psm1` and `.psd1` in non-discovery searches); on Windows every `PATHEXT` extension when applications are allowed; then the bare name if not yet added. A `.ps1` file becomes an external script; any other existing file becomes an application. If the normal search fails and the name contains neither `-` nor `\`, the lookup is repeated once with the name prefixed with `Get-`. If still unresolved, module-qualified names trigger module auto-loading, and (with `$PSModuleAutoLoadingPreference` = `All`, the default) other names trigger module auto-discovery against the module analysis cache; then the *command-not-found* handler runs; then a `CommandNotFoundException` is raised. In a session whose origin is `Runspace`, application and external-script types are removed from the search when the session state has no allowed applications or scripts, and a name with a directory part must match an allow-list entry (prefix match, or the list is exactly `*`). + +**Invocation after lookup** (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.LookupCommandProcessor@L276-L291): (1) follow an alias chain while the item is an alias that is visible to the caller (a `HashSet` of names already seen stops loops); an alias whose target cannot be resolved raises `AliasNotResolvedException`; (2) `ShouldRun`: for `Runspace` origin a non-public command is reported as not found; then the authorization manager (`ShouldRunInternal`: execution policy / signature rules) is asked, and a `PSSecurityException` is logged as a health event and as a terminated command before rethrow; (3) create the processor by type (§4); (4) set `CommandOrigin` and `InvocationName` on the command. + +**External scripts**: `SignatureChecked` is set before creating the processor; `VerifyScriptRequirements` then checks, in order, elevation, `#requires -Version`, `#requires -PSEdition`, and required modules (loaded with `LoadRequiredModule`; a failure raises `ScriptRequiresMissingModules` unless the lookup is for completion); a script that names a `#requires -ShellId` application id fails with `RequiresShellIDInvalidForSingleShell` (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.CreateScriptProcessorForSingleShell@L323-L340). + +## 7. Algorithms + +**The searcher state machine** (src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239): states `SearchingAliases` → `SearchingFunctions` → `SearchingCmdlets` → `StartSearchingForExternalCommands` → `PowerShellPathResolution` → `QualifiedFileSystemPath` → `PathSearch` → `PowerShellRelativePath` → `NoMoreMatches`. Each call continues from the saved state, so the same enumerator can yield several matches (wildcard or fuzzy searches, as used by `Get-Command`). Without pattern options, alias, function and cmdlet steps yield at most one match each and then advance the state. + +**Constrained-language filter** (src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.ShouldSkipCommandResolutionForConstrainedLanguage@L836-L876): a found command is hidden (treated as not found) when it was defined in `ConstrainedLanguage` and the session now runs in `FullLanguage` — unless the system lockdown policy is `Audit`, in which case it is logged and kept; a function defined in `FullLanguage` is hidden when the session is in `ConstrainedLanguage` *and the debugger is stopped at a breakpoint* and the function is not in the global scope table. The filter is applied to alias, function and file results. + +**Native argument building** (src/System.Management.Automation/engine/NativeCommandParameterBinder.cs::NativeCommandParameterBinder.BindParameters@L70-L130, `AppendOneNativeArgument`, `PossiblyGlobArg`): arguments are rendered both as one command-line string (`Arguments`) and as a list (`ArgumentList`); which one is used depends on the *argument-passing style* variable `$PSNativeCommandArgumentPassing` (`Legacy`, `Standard`, `Windows`; the variable is initialized to `Windows` on Windows and `Standard` on Unix, and an unset variable reads as `Standard`; an unconvertible value falls back to `Legacy`) (src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.GetPassingStyle@L4675-L4682) (src/System.Management.Automation/engine/NativeCommandParameterBinder.cs::NativeCommandParameterBinder.ArgumentPassingStyle@L192-L208): `Legacy` → always the string; `Windows` → the string only for files with extension `.js .wsf .cmd .bat .vbs` or commands named `cmd cscript find sqlcmd wscript`, otherwise the list; `Standard` → the list (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.UseSpecialArgumentPassing@L1586-L1592). Rules read: an enumerable argument is expanded, one element per argument (elements of a comma list written in source are re-joined with the separator text used in the source: `,`, `, `, ` ,` or ` , `; elements of other enumerables are joined with a space); an empty-string argument is dropped in `Legacy` and passed as an empty argument otherwise; after the `--%` marker the remaining text is passed verbatim (environment variables expanded); an argument needs quotes when it contains whitespace outside balanced quotes; an unquoted argument containing wildcard characters is expanded on Unix when the current location is the file system and every match is a file-system item; an unquoted `~` or `~/…` is replaced by the file-system provider's home; bare words keep `-name:value` form (`name:` and value are joined unless a space followed the colon). + +**Redirection decision** (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.CalculateIORedirection@L1762-L1884): input is redirected when the command expects pipeline input; output and error are redirected by default; if the command is last in the pipeline and its downstream is `Out-Default` (without `-Transcript`) and no explicit stdout destination is set, output is *not* redirected (the child writes straight to the console); error is not redirected when it is not merged into output and its downstream is `Out-Default`; for server-side (remoting) hosts everything is redirected; on Windows a non-Windows-GUI program forces a hidden console. If none of the three streams is redirected the process runs *stand-alone*. When transcription is on and the host cannot read the screen buffer, output and error are redirected after all. + +## 8. Observable behavior [SPEC] + +- A function or alias shadows a cmdlet of the same name; a cmdlet shadows a file on `PATH`; `Module\Cmdlet` forces a module. +- An unknown name that is not a path first tries `Get-`, so `process` can run `Get-Process`. +- Running a program whose exit code is non-zero sets `$LASTEXITCODE` and marks the pipeline's `$?` false; an error is raised only when `$PSNativeCommandUseErrorActionPreference` is true (the error id is `ProgramExitedWithNonZeroCode`, category `NotSpecified`, carrying path, exit code and process id). +- A program in the middle of a pipeline has its output captured and delivered as objects (text lines); a program last in the pipeline into `Out-Default` writes directly to the console. +- Tests that specify these behaviors are listed in §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +- Errors during search that are *not* "not found" (bad characters, path too long, bad file format, metadata errors) are remembered as `lastError` and attached as the inner exception of the final `CommandNotFoundException` (`TryNormalSearch`). +- Event handlers (pre/post lookup, command-not-found) are invoked with the exception swallowed; a re-entrancy guard (`RegisterLookupCommandInfoAction`) prevents a handler's own lookups from invoking the same handler again. +- Auto-loading raises `CouldNotAutoLoadModule` / `CouldNotAutoloadMatchingModule` as `CommandNotFoundException`; other exceptions during auto-discovery are swallowed. +- Starting a process: a `Win32Exception` is retried on Windows through the file-association executable; a solo command may be re-tried with shell execute and no redirection; otherwise `ApplicationFailedException` is thrown with the command name, the cause and the position message. A document (non-executable) cannot run on Nano/IoT, and cannot run inside a pipeline (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.GetProcessStartInfo@L1602-L1712). +- If the pipeline is stopping before the process starts, `PipelineStoppedException`. + +## 10. Concurrency and threading + +The searcher and discovery object are used on the pipeline thread; the `PATH` and `PATHEXT` caches use a static lock for `PATHEXT`. The native processor has a lock (`_sync`) protecting process start and `_stopped`; `StopProcessing` can be called from another thread: it sets `_stopped`, stops the input writer, disposes the byte-transfer and kills the process (only when not running stand-alone). Output from a redirected process is consumed through a queue filled on process-output callbacks and drained on the pipeline thread (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.ConsumeAvailableNativeProcessOutput@L1044-L1080). + +**Process lifecycle** (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Prepare@L557-L579, `ProcessRecord`, `Complete`): *the process is started in the prepare step* (before any begin block runs), after the arguments are bound and I/O redirection is decided; `ProcessRecord` feeds each pipeline object to the child's input writer (not when the upstream command is also native — then bytes are connected directly); `Complete` closes the input, reads remaining output until the process ends, waits for exit, stores the exit code in `$LASTEXITCODE` and applies the error rule of §8; clean-up always runs and restores the host (`NotifyBeginApplication` / end-application notifications). The last program in a pipeline is treated as a *background* program (not waited for) when it was started through shell execute or is a Windows GUI-subsystem application; otherwise it is waited for. Clean-up reports end-of-application to the host, and on Unix kills a still-running child unless it is a background program that was not asked to be killed. + +## 11. Extension points and customization hooks + +`$ExecutionContext.InvokeCommand.PreCommandLookupAction`, `PostCommandLookupAction` and `CommandNotFoundAction` (event arguments carry `CommandName`, `CommandOrigin`, `CommandScriptBlock`/`Command`, `StopSearch`); `$PSModuleAutoLoadingPreference` (`None`, `ModuleQualified`, `All`) and the environment fallback; `$PSNativeCommandArgumentPassing`; `$PSNativeCommandUseErrorActionPreference`; `$PSDefaultParameterValues` (applied at binding); session-state `Applications` / `Scripts` allow-lists and command visibility in constrained runspaces (`InitialSessionState` entries). + +## 12. Dependencies + +Inbound: pipeline creation (`Command.CreateCommandProcessor`, `LocalPipeline`), script command invocations in compiled code, `Get-Command`, completion. Outbound: session state tables (T29), `AnalysisCache`/module loader (T37), `AuthorizationManager` (execution policy), `LocationGlobber`/providers (T34) for path resolution, `PSObject.ToStringParser` for argument text, the host (`NotifyBeginApplication`), `Console.OutputEncoding` for process stream encoding. + +## 13. Performance notes + +`PATH` and `PATHEXT` splits are cached by raw string; cmdlet instances are created through cached compiled factories (see [runspace-and-pipeline](runspace-and-pipeline.md) §13); auto-discovery walks every module file found under `PSModulePath` through the analysis cache only after normal search and module-qualified loading failed, so the cost lands on *misspelled or unknown* names. Pattern searches materialize all matching aliases/functions/cmdlets before enumerating. + +## 14. Security considerations + +Trust assumptions: the name resolution result decides what code runs, so everything that can add an alias, function or `PATH` entry can redirect commands (that is why trusted runspaces limit `Applications`/`Scripts`, and why a name with a directory part is checked against allow-lists in `Runspace` origin). Enforced at lookup: (a) visibility for `Runspace` origin (src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.ShouldRun@L538-L564); (b) the authorization manager (execution policy and signatures) for every command that is run; (c) hiding of commands defined in a *less* trusted language mode when the session is in a more trusted one (src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.ShouldSkipCommandResolutionForConstrainedLanguage@L836-L876, with audit mode logging a WDAC message); (d) module auto-loading only when `Import-Module` (and `Get-Module` for discovery) is visible to the caller; (e) `#requires` checks for scripts. Native commands run with the engine's own privileges and environment; the working directory is the file-system provider's current location; Windows adds `PSModulePath` for Windows PowerShell child processes. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 22 | +| app-control | 4 | +| logging | 1 | +| remoting-deser | 1 | +| **total** | 28 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/ScriptCommandProcessor.cs` | 13 | +| `src/System.Management.Automation/engine/CommandSearcher.cs` | 9 | +| `src/System.Management.Automation/engine/CommandInfo.cs` | 4 | +| `src/System.Management.Automation/engine/NativeCommandProcessor.cs` | 2 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: command identity is resolved once, with the resolved *path or definition* recorded in the audit trail; policy decisions (allow/deny) are taken on the resolved identity, not on the typed name; implicit behaviors that widen the search (the `Get-` prefix retry and module auto-discovery) are individually switchable by policy; hiding commands by trust level is enforced at lookup *and* at invocation of an already-held `CommandInfo`. + +## 15. Enterprise considerations + +Security first: §14. Observability: ETW events for lookup start/stop, module auto-discovery and auto-loading; tracer `discoveryTracer`; command lifecycle events from the pipeline. Supportability: the error ids (`CommandNotFoundException`, `AliasNotResolvedException`, `CouldNotAutoLoadModule`, `ScriptRequires*`) are specific. Testability: `CommandSearcher` is an enumerator and can be exercised alone, but needs an `ExecutionContext`. Resource limits: none on lookup time; discovery scans may be slow on large module paths. Missing: a resolution *trace* result for diagnostics (the tracer is developer-facing), policy hooks per resolution step, signed command catalogs. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandArguments.Tests.ps1` | 5 | 0 | 15 | +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandPathUpdate.Tests.ps1` | 1 | 2 | 9 | +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandProcessor.Tests.ps1` | 6 | 0 | 15 | +| `test/powershell/Language/Scripting/NativeExecution/NativeLinuxCommands.Tests.ps1` | 2 | 0 | 8 | +| `test/powershell/Language/Scripting/NativeExecution/NativeStreams.Tests.ps1` | 2 | 1 | 6 | +| `test/powershell/Language/Scripting/NativeExecution/NativeUnixGlobbing.Tests.ps1` | 1 | 0 | 17 | +| `test/powershell/Language/Scripting/NativeExecution/NativeWindowsTildeExpansion.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/Language/Scripting/Requires.Tests.ps1` | 2 | 7 | 8 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Get-Command.Tests.ps1` | 1 | 0 | 16 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Command.Tests.ps1` | 2 | 4 | 17 | +| `test/powershell/engine/Basic/CommandDiscovery.Tests.ps1` | 1 | 4 | 15 | +| `test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1` | 1 | 0 | 8 | +| `test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1` | 1 | 2 | 10 | +| **total (13 files)** | 26 | 20 | 147 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `engine/Basic/CommandDiscovery`, `NativeExecution/`, `engine/Basic/NativeCommand`, `Microsoft.PowerShell.Core/Get-Command`, `Microsoft.PowerShell.Utility/Get-Command`, `Scripting/Requires`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: aliases are searched before functions, which are searched before cmdlets — user-level definitions override built-ins by design (`INFERENCE` on intent). +- `FACT-SRC`: the file search is the last step; an application name never beats an in-session command. +- `INFERENCE`: the `Get-` prefix retry exists so that short names (`date`, `process`) work for verbs-nouns; it makes failed lookups cost two searches. +- `FACT-SRC`: three argument-passing styles exist because argument quoting changed incompatibly; the default is `Standard` on Unix and `Windows` on Windows (a hybrid: `Standard` except for a fixed list of legacy programs), with `Legacy` still available. +- `FACT-SRC`: the native process starts at prepare time, so a program in a pipeline begins running before earlier commands have run their begin blocks. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTDISC-01** [REPLICATE] (area: NLD-CMD) Resolution order alias → function → cmdlet → path/file, with module-qualified names bypassing precedence; it is observable language behavior. Evidence: src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239. +- **LSN-RTDISC-02** [REPLICATE] (area: NLD-CMD) Make the searcher a resumable enumerator so `Get-Command`-style wildcard queries share the code with single lookups. Evidence: src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher@L22-L1708. +- **LSN-RTDISC-03** [ADAPT] (area: NLD-SEC) Keep the lookup events (pre, not-found, post) but give them typed results and a re-entrancy guard in the type system instead of a name→set dictionary of strings. Evidence: src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.RegisterLookupCommandInfoAction@L1156-L1169. +- **LSN-RTDISC-04** [REPLICATE] (area: NLD-SEC) Visibility and trust-level filtering belong in the lookup, not only at run time. Evidence: src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.ShouldSkipCommandResolutionForConstrainedLanguage@L836-L876. +- **LSN-RTDISC-05** [REDESIGN] (area: NLD-CMD) Replace module auto-discovery by file analysis on every unknown name with an explicit, signed command index that is refreshed on module install/update; keep module-qualified auto-load. DEV: DEV-DISC-120. Evidence: src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.TryModuleAutoDiscovery@L951-L1055. +- **LSN-RTDISC-06** [AVOID] (area: NLD-CMD) Do not implement the `Legacy` native-argument passing style (string re-quoting) or the name-based legacy-program list; offer only list-based passing (`Standard`) and a per-command opt-in for programs that need a raw command line. DEV: DEV-DISC-121. Evidence: src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.ShouldUseLegacyPassingStyle@L1720-L1729. +- **LSN-RTDISC-07** [ADAPT] (area: NLD-CMD) Start the child process after all begin phases (or make the start point explicit) so pipeline ordering is the same for cmdlets and programs; keep the "console-direct when last into the default output" optimization. Evidence: src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Prepare@L557-L579. +- **LSN-RTDISC-08** [REPLICATE] (area: NLD-CMD) `$LASTEXITCODE` set on every native exit; error-on-non-zero as an opt-in preference. Evidence: src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-027): `CommandPathSearch` internals (directory enumeration, ordering inside a directory, case rules, caching), `CommandInfo` subclasses other than their use here, `GetCommandCommand` (how pattern/abbreviation options are used). +- `UNVERIFIED` (VB-028): `NativeCommandProcessor` parts not read: `InitOutputQueue`/`ProcessOutputRecord` (decoding and line splitting), `KillProcess*`, byte-pipe classes (`BytePipe`, `AsyncByteStreamTransfer`), `MinishellParameterBinderController`, package-manager path update (Windows). +- `UNVERIFIED` (VB-029): the `AuthorizationManager.ShouldRunInternal` algorithm (execution policy; Phase 5). + +Verification: all non-comment lines of `CommandSearcher.cs` lines 22-1100 and 1322-1640, `CommandDiscovery.cs` lines 276-660, 862-1260, 1284-1464, `NativeCommandProcessor.cs` lines 557-606, 715-967, 1082-1237, 1449-1475, 1586-1884 and `NativeCommandParameterBinder.cs` lines 58-470 were read; ranges are in ledger proposal `T31_T39.csv`. The `CommandTypes` members were checked against source (no `Workflow`). Test counts: `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/03-runtime/debugger.md b/docs-out/03-runtime/debugger.md new file mode 100644 index 0000000..206e4da --- /dev/null +++ b/docs-out/03-runtime/debugger.md @@ -0,0 +1,185 @@ +--- +id: RT-DEBUGGER-001 +title: Debugger +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/debugger/debugger.cs, src/System.Management.Automation/engine/debugger/Breakpoint.cs, src/System.Management.Automation/engine/CommandProcessorBase.cs, src/System.Management.Automation/engine/CommandProcessor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/debugger/_folder.md] +generated_from: [data/types.json, data/enums.json, data/tests.json, data/enforcement-points.json] +--- + +# Debugger + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. Where sequence points come from (compiler) and how the interpreter reaches the debugger hook: [02.5-compiler-and-engine](../02-language/02.5-compiler-and-engine.md) and [TIERING-POLICY](../02-language/TIERING-POLICY.md) (debugging forces the unoptimized variant). + +## 1. Purpose and scope [SPEC] + +The debugger lets a host stop running script at breakpoints or steps, inspect and change state, and resume. This document covers the debugger object model (`Debugger`, `ScriptDebugger`, nested/runspace debuggers), breakpoint kinds, the engine hooks that trigger it, the resume actions and the command set at the debugger prompt. Remote runspace debugging and job debugging are noted but not analysed. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/debugger/debugger.cs::Debugger@L320-L966 | abstract class | public debugger API: modes, breakpoints, `ProcessCommand`, `SetDebuggerAction`, events | B | +| src/System.Management.Automation/engine/debugger/debugger.cs::ScriptDebugger@L975-L4218 | sealed class | the local script debugger: call stack, breakpoint tables, stepping, stop handling | B | +| src/System.Management.Automation/engine/debugger/debugger.cs::NestedRunspaceDebugger@L4227-L4629 | abstract class | debugging another runspace/job from the current one (`Standalone…`, `Embedded…`) | B | +| src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerCommandProcessor@L5085-L5419 | class | the prompt command table (`s`, `v`, `o`, `c`, `q`, `d`, `k`, `l`, `h`) | B | +| src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerStopEventArgs@L58-L115 | class | data of the stop event: invocation info, breakpoints, resume action | B | +| src/System.Management.Automation/engine/debugger/Breakpoint.cs::Breakpoint@L16-L150 | abstract class | base breakpoint: id, script, action, enabled, hit count | B | +| src/System.Management.Automation/engine/debugger/Breakpoint.cs::LineBreakpoint@L356-L611 | class | line and column breakpoints | B | +| src/System.Management.Automation/engine/debugger/Breakpoint.cs::CommandBreakpoint@L155-L252 | class | breakpoint on a command name (wildcards) | B | +| src/System.Management.Automation/engine/debugger/Breakpoint.cs::VariableBreakpoint@L276-L351 | class | breakpoint on variable read/write/both | B | + +## 3. Responsibilities and non-responsibilities + +Owns: breakpoint tables per script/function and their binding to sequence points, the stepping state machine, the stop/resume protocol, debugger commands, call-stack frames. Does not own: sequence point generation (compiler), the interactive prompt UI (host `PSHost.EnterNestedPrompt`), the `*-PSBreakpoint` cmdlets (command-module family), remoting transport. + +## 4. Key types and contracts [IMPL] + +- **`DebuggerResumeAction`**: `Continue=0`, `StepInto=1`, `StepOut=2`, `StepOver=3`, `Stop=4` (src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerResumeAction@L31-L53). **`DebugModes`** flags: `None=0`, `Default=1`, `LocalScript=2`, `RemoteScript=4`. +- **Debugger API**: `IsActive`, `InBreakpoint`, `ProcessCommand(PSCommand, output)` (run a command in the stopped scope), `SetDebuggerAction`, `GetDebuggerStopArgs`, `StopProcessCommand`, `SetDebugMode`, `GetCallStack`, `Get/SetBreakpoints`, `SetCommandBreakpoint`, `SetLineBreakpoint`, `SetVariableBreakpoint`, `Remove/Enable/DisableBreakpoint`, `SetDebuggerStepMode`, `CancelDebuggerProcessing`; events `DebuggerStop`, `BreakpointUpdated`; internal `Break(triggerObject)` to enter the debugger from code (used for `Break` action preferences) and `DebugJob`/`DebugRunspace` for nested debugging. +- **Breakpoint kinds**: line (script path, line, optional column), command (name pattern, optional script scope), variable (name, access mode read/write/read-write); each has an optional **action** script block that decides whether to stop (a `break` in the action stops; other results continue), an enabled flag and a hit counter (`UNVERIFIED` details). +- **Engine hooks** (called by compiled code and command processors only when `ExecutionContext._debuggingMode` is non-zero): `EnterScriptFunction(functionContext)` / `ExitScriptFunction()` maintain the debugger's call stack of function contexts; `OnSequencePointHit(functionContext)` is called at each sequence point; `CheckCommand(invocationInfo)` is called before a command's `BeginProcessing` (for non-script cmdlets: `CommandProcessorBase.DoBegin` and `CommandProcessor.ProcessRecord` check `_debuggingMode > 0 && Command is not PSScriptCmdlet`); `CheckVariableRead/Write(name)` on variable access when variable breakpoints exist. +- **`OnSequencePointHit` algorithm** (src/System.Management.Automation/engine/debugger/debugger.cs::ScriptDebugger.OnSequencePointHit@L1537-L1604): (1) if statement tracing is on (`Set-PSDebug -Trace`), and the frame is not hidden and the function is not marked step-through and the position is a real position, print the trace line; (2) if a nested debugger stop was requested, stop now; (3) update breakpoint bindings for this function context (`UpdateBreakpoints`); (4) if the stepping mode is *step in* and the current frame is the one being stepped over/out (or none), stop on this sequence point unless the frame is hidden (a hidden frame during step-in resumes with *step out*); (5) otherwise, if the function's breakpoint bit set marks this sequence point, take the *enabled* line breakpoints bound to it, run `TriggerBreakpoints` (conditions/actions), and stop with the survivors. `CheckCommand` filters enabled command breakpoints by `Trigger(invocationInfo)`, triggers them, and if any survive stops with the function's current position and tells the caller to skip line-breakpoint checks for that command (src/System.Management.Automation/engine/debugger/debugger.cs::ScriptDebugger.CheckCommand@L1389-L1417). +- **Prompt commands** (`DebuggerCommandProcessor`): `continue`/`c`, `stepInto`/`s` (repeat on empty Enter), `stepOver`/`v` (repeat on Enter), `stepOut`/`o`, `quit`/`q` (stop), `detach`/`d`, `k` (= `Get-PSCallStack`), `list`/`l` (default 16 lines, repeats on Enter), `h`/`?` (help); everything else is run as script in the stopped scope; the empty command repeats the last repeatable command. Resume actions map to `DebuggerResumeAction` values. + +## 5. Data model + +`ScriptDebugger` holds: the call stack (`CallStackFrame`s with function context, position, hidden flag), the command-breakpoint table, per-script line-breakpoint tables, the current stepping mode (`StepIn`, over/out frame reference), the current debugger action, the pending-stop state, and an event for the host. A `FunctionContext` carries `_breakPoints` (bit array indexed by sequence point), `_boundBreakpoints` (sequence point → breakpoints), `_currentSequencePointIndex`, `_debuggerHidden` and `_debuggerStepThrough` flags. `PSDebugContext` is the object bound to `$PSDebugContext` while stopped (invocation info and breakpoints). + +## 6. Control flow + +```mermaid +sequenceDiagram + participant C as Compiled code or interpreter + participant D as ScriptDebugger + participant H as Host (DebuggerStop handler) + C->>D: EnterScriptFunction + C->>D: OnSequencePointHit at each statement + D->>D: trace, update bindings, stepping or breakpoint check + D->>H: DebuggerStop event with DebuggerStopEventArgs + H->>D: ProcessCommand (prompt input) + H->>D: SetDebuggerAction (Continue, StepInto, StepOver, StepOut, Stop) + D->>C: resume or throw to stop + C->>D: ExitScriptFunction +``` + +Ordered steps at a stop: freeze the pipeline, raise `DebuggerStop` (or enter a nested prompt on the host), read commands until a resume action is set, apply the action (stepping modes set the frame reference), continue. `UNVERIFIED`: the locking and the exact way `Stop` unwinds (via `ScriptCallDepth`/`PipelineStopped` style exceptions). + +## 7. Algorithms + +Stepping: *step in* stops at the next sequence point in any non-hidden frame; *step over* sets the over/out frame to the current frame and stops at the next sequence point of that frame (or a shallower one); *step out* stops after the current frame returns; hidden frames are skipped by resuming with *step out* (`INFERENCE` from the code above). Breakpoint binding: line breakpoints are resolved to sequence points when the script block is compiled/entered (`UpdateBreakpoints`), stored as a bit in `_breakPoints` for O(1) checks per sequence point. + +## 8. Observable behavior [SPEC] + +- `Set-PSBreakpoint -Script -Line` stops before the statement at that line; `-Command` stops when the command is about to run; `-Variable` stops on access; `-Action` runs a script block that can `break`. +- At the `DBG>` prompt, `s`, `v`, `o`, `c`, `q`, `k`, `l`, `h` behave as above; any other input runs in the stopped scope. +- Debugging is only active in runspaces where debug mode is enabled; debugging forces the slower (unoptimized) script execution variant. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +A breakpoint action that throws is reported; nested debugger stop requests are queued; debugging a constrained-language session is limited (`ConstrainedLanguageDebugger.Tests.ps1`: commands typed at the prompt are restricted); hidden frames (`DebuggerHidden` attribute) are skipped; `IsPendingDebugStopEvent` marks a stop requested from another thread; the debugger refuses invalid transitions (for example resume while not stopped; `UNVERIFIED` messages). + +## 10. Concurrency and threading + +Stop events are raised on the pipeline thread that hit the breakpoint; `Break`-all and `Debug-Runspace`/`Enable-RunspaceDebug` request a stop asynchronously from another thread, so `ScriptDebugger` keeps pending-stop state and synchronization primitives (`UNVERIFIED` details). Nested debugger types serialize commands through a processing lock. + +## 11. Extension points and customization hooks + +Host `DebuggerStop` handler / nested prompt; breakpoint `-Action`; `Set-PSDebug -Trace/-Step/-Strict`; `Wait-Debugger`; `[System.Diagnostics.DebuggerHidden()]`/`DebuggerStepThrough` attributes on functions; `Enable-RunspaceDebug`; `Debug-Job`/`Debug-Runspace`; `PSDebugContext`/`$PSDebugContext` in the prompt; custom debugger implementations in hosts (public abstract `Debugger`). + +## 12. Dependencies + +Inbound: compiled code and interpreter (sequence points), command processors (`CheckCommand`), variable access (variable breakpoints), `Warning`/`Error` preference `Break`, jobs (`AtBreakpoint` state), runspaces (`Runspace.Debugger`). Outbound: host UI (prompt), script parser positions, `PowerShell` API (`ProcessCommand` runs commands), remoting for remote debugging. + +## 13. Performance notes + +When no breakpoints or tracing exist the hook is not called (`_debuggingMode == 0`), and the optimized variant is used; when debugging is enabled the unoptimized variant runs and every sequence point calls into the debugger (O(1) bit check). Breakpoint tables are rebuilt per function entry. + +## 14. Security considerations + +The debugger is a code-execution and state-inspection channel: at a stop, arbitrary commands run in the stopped scope with its variables; a remote debugger can attach to another runspace. Enforcement seen: debugging in constrained language mode restricts what can be run at the prompt (test `ConstrainedLanguageDebugger`, enforcement location `UNVERIFIED`); a function defined in full language is hidden from command resolution while stopped in a constrained session (see [command-discovery-and-invocation](command-discovery-and-invocation.md) §7); `CommandProcessorBase.DoBegin` only calls the hook for non-script cmdlets. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 6 | +| app-control | 1 | +| **total** | 7 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/debugger/debugger.cs` | 7 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: attaching a debugger requires an explicit privilege and is audited; the debug prompt runs under the *lowest* language mode in the stack; breakpoint actions run under the mode of their definer; debug mode can be disabled by policy and then no hook costs or state exist. + +## 15. Enterprise considerations + +Security first: §14. Observability: breakpoint hit counts, trace lines, `$PSDebugContext`, call stack (`Get-PSCallStack`). Supportability: standard prompt commands and listing. Testability: the SDK exposes `Debugger` for automated tests (see SDK tests). Resource limits: none on breakpoint count. Missing: audit log of debug sessions, per-user permission model, non-interactive debug adapters (DAP is external). + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Scripting/Debugging/DebuggerCommand.Tests.ps1` | 3 | 9 | 51 | +| `test/powershell/Language/Scripting/Debugging/DebuggerScriptTests.Tests.ps1` | 9 | 1 | 22 | +| `test/powershell/Language/Scripting/Debugging/Debugging.Tests.ps1` | 3 | 9 | 19 | +| `test/powershell/Language/Scripting/Debugging/DebuggingInHost.Tests.ps1` | 1 | 0 | 4 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Set-PSDebug.Tests.ps1` | 1 | 1 | 3 | +| `test/powershell/Modules/Microsoft.PowerShell.Security/ConstrainedLanguageDebugger.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Debug-Runspace.Tests.ps1` | 1 | 0 | 4 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Disable-PSBreakpoint.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Enable-PSBreakpoint.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-PSBreakpoint.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-RunspaceDebug.Tests.ps1` | 1 | 1 | 1 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Remove-PSBreakpoint.Tests.ps1` | 1 | 1 | 7 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/RunspaceBreakpointManagement.Tests.ps1` | 1 | 3 | 20 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Set-PSBreakpoint.Tests.ps1` | 2 | 0 | 24 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Wait-Debugger.Tests.ps1` | 1 | 1 | 2 | +| `test/powershell/SDK/Breakpoint.Tests.ps1` | 1 | 5 | 35 | +| `test/powershell/SDK/PSDebugging.Tests.ps1` | 2 | 2 | 12 | +| `test/powershell/engine/Remoting/InvokeCommandRemoteDebug.Tests.ps1` | 1 | 0 | 4 | +| **total (18 files)** | 32 | 33 | 216 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `ConstrainedLanguageDebugger`, `Set-PSDebug`, `Disable-PSBreakpoint`, `Enable-PSBreakpoint`, `Remove-PSBreakpoint`, `Get-RunspaceDebug`, `RunspaceBreakpointManagement`, `Debug-Runspace`, `Get-PSBreakpoint`, `Wait-Debugger`, `Set-PSBreakpoint`, `SDK/Breakpoint`, `SDK/PSDebugging`, `InvokeCommandRemoteDebug`, `Scripting/Debugging/`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: debugging works on sequence points with a bit array per function, so the disabled cost is a flag check and the enabled cost per statement is small. +- `FACT-SRC`: the debugger is a public abstract class so hosts can replace it. +- `INFERENCE`: restricting hooks to non-script cmdlets avoids double-stopping for script commands (their sequence points already stop). +- `FACT-SRC`: two prompt commands repeat on Enter (step into/over) to speed stepping. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTDBG-01** [REPLICATE] (area: NLD-TOOL) Sequence points with per-function bit arrays and a single hook called only in debug mode. Evidence: src/System.Management.Automation/engine/debugger/debugger.cs::ScriptDebugger.OnSequencePointHit@L1537-L1604. +- **LSN-RTDBG-02** [REPLICATE] (area: NLD-TOOL) Three breakpoint kinds (line, command, variable) with action blocks, and the five resume actions. Evidence: src/System.Management.Automation/engine/debugger/Breakpoint.cs::Breakpoint@L16-L150. +- **LSN-RTDBG-03** [ADAPT] (area: NLD-TOOL) Expose the debugger through a protocol-neutral interface suited to an external adapter (the Debug Adapter Protocol), keeping the prompt commands for terminals. Evidence: src/System.Management.Automation/engine/debugger/debugger.cs::Debugger@L320-L966. +- **LSN-RTDBG-04** [REDESIGN] (area: NLD-SEC) Debugger attach/enable is a policy-controlled, audited privilege, and the prompt runs under the lowest trust level present on the stack. DEV: DEV-DBG-195. Evidence: src/System.Management.Automation/engine/debugger/debugger.cs::NestedRunspaceDebugger@L4227-L4629. +- **LSN-RTDBG-05** [AVOID] (area: NLD-TOOL) Do not port nested debugging of jobs and remote runspaces (`DebugJob`, `Embedded/StandaloneRunspaceDebugger`) in the first version. DEV: DEV-DBG-196. Evidence: src/System.Management.Automation/engine/debugger/debugger.cs::StandaloneRunspaceDebugger@L4636-L4729. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-053): `ScriptDebugger` stop/resume machinery (lines 1700–2400, 2400–4200): `OnDebuggerStop`, `ResumeExecution`, locking, `ProcessCommand`, remote and nested debuggers. +- `UNVERIFIED` (VB-054): breakpoint classes' trigger rules (`LineBreakpoint.Trigger`, wildcard rules of `CommandBreakpoint`), `BreakpointUpdated` events, `UpdateBreakpoints` binding algorithm, `PSDebugContext`, `CallStackFrame`. +- `UNVERIFIED` (VB-055): the cmdlets `Set/Get/Remove/Enable/Disable-PSBreakpoint`, `Debug-Runspace`, `Wait-Debugger`, and the host prompt integration. + +Verification: `debugger.cs` lines 31–60 (enum values), 320–460 (outline by grep), 1389–1440, 1537–1600 and the member list by grep; `DebuggerCommandProcessor` lines 5085–5160 were read; other ranges are from signatures only. Ranges in `T31_T39.csv`. diff --git a/docs-out/03-runtime/errors-and-diagnostics.md b/docs-out/03-runtime/errors-and-diagnostics.md new file mode 100644 index 0000000..ec47ce6 --- /dev/null +++ b/docs-out/03-runtime/errors-and-diagnostics.md @@ -0,0 +1,193 @@ +--- +id: RT-ERRORS-001 +title: Errors and diagnostics +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-ERR-001] +sources: [src/System.Management.Automation/engine/ErrorPackage.cs, src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs, src/System.Management.Automation/engine/ExecutionContext.cs, src/System.Management.Automation/engine/QuestionMarkVariable.cs, src/System.Management.Automation/engine/MshCommandRuntime.cs, src/System.Management.Automation/engine/CommandProcessorBase.cs, src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs, src/System.Management.Automation/engine/NativeCommandProcessor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/types.json, data/enums.json, data/tests.json, data/enforcement-points.json] +--- + +# Errors and diagnostics + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. The statement-level protocol (`CheckActionPreference`, traps, try/catch lowering) is documented where it is implemented: [RUNTIME-OPERATIONS](../02-language/RUNTIME-OPERATIONS.md) §6.1 and [02.5-compiler-and-engine](../02-language/02.5-compiler-and-engine.md) §6.2–6.4. Cmdlet-side stream behavior: [cmdlet-infrastructure](cmdlet-infrastructure.md). Pipeline first-error rule: [runspace-and-pipeline](runspace-and-pipeline.md). Parse errors: [ERROR-CATALOG](../02-language/ERROR-CATALOG.md). + +## 1. Purpose and scope [SPEC] + +How failures are represented, reported and observed: the `ErrorRecord` data model, the exception hierarchy, the `$Error` history, `$?`, `$LASTEXITCODE`, `$ErrorActionPreference` and `-ErrorAction`, the error views, and transcripts. This document ties the per-layer rules together and owns the canonical statement of terminating versus non-terminating errors (CF-ERR-001). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/ErrorPackage.cs::ErrorRecord@L988-L1699 | class | the error data model | B | +| src/System.Management.Automation/engine/ErrorPackage.cs::ErrorDetails@L530-L962 | class | optional replacement message, recommended action | B | +| src/System.Management.Automation/engine/ErrorPackage.cs::ErrorCategory@L28-L189 | enum | 32 categories | B | +| src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps | static class | statement-level handling, traps, `$?` | B | +| src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L818-L858 | method | `$Error` maintenance | B | +| src/System.Management.Automation/engine/QuestionMarkVariable.cs::QuestionMarkVariable@L9-L42 | class | the `$?` variable | B | +| src/System.Management.Automation/engine/MshCommandRuntime.cs | class | cmdlet-side error writing and variables | B | +| src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs::TranscriptionData@L1190-L1210 | class | transcript state | B | +| src/System.Management.Automation/engine/NativeCommandProcessor.cs | class | `$LASTEXITCODE` and native exit rule | A | + +## 3. Responsibilities and non-responsibilities + +Owns: the error model, history, status variables, view enumeration, the combined terminating/non-terminating rules. Does not own: parse error ids (T22), the per-statement compiled protocol (T25/T26, referenced), the console renderer of error views (formatting, T36), the `Start-Transcript` cmdlet (host family). + +## 4. Key types and contracts [IMPL] + +- **`ErrorRecord`** wraps: `Exception`, `TargetObject`, `CategoryInfo` (category, activity, reason, target name/type), `FullyQualifiedErrorId`, `ErrorDetails` (replacement message, recommended action), `InvocationInfo` (where the error was raised), `ScriptStackTrace`, `PipelineIterationInfo` and a "serialized" form used for remoting (`_serializedFullyQualifiedErrorId`, deserialization constructor). A `ParentContainsErrorRecordException` wraps an exception around an existing record. +- **`ErrorCategory`** (32 values, numeric order): `NotSpecified=0`, `OpenError`, `CloseError`, `DeviceError`, `DeadlockDetected`, `InvalidArgument`, `InvalidData`, `InvalidOperation`, `InvalidResult`, `InvalidType`, `MetadataError`, `NotImplemented`, `NotInstalled`, `ObjectNotFound`, `OperationStopped`, `OperationTimeout`, `SyntaxError`, `ParserError`, `PermissionDenied`, `ResourceBusy`, `ResourceExists`, `ResourceUnavailable`, `ReadError`, `WriteError`, `FromStdErr`, `SecurityError`, `ProtocolError`, `ConnectionError`, `AuthenticationError`, `LimitsExceeded`, `QuotaExceeded`, `NotEnabled` (src/System.Management.Automation/engine/ErrorPackage.cs::ErrorCategory@L28-L189). +- **Exception hierarchy** (from the type scan, direct bases): `RuntimeException` is the engine's root script-visible exception; its direct subclasses include `CmdletInvocationException`, `ParameterBindingException`, `ParseException`, `CommandNotFoundException`, `ProviderInvocationException`, `SessionStateException`, `MetadataException`, `ExtendedTypeSystemException`, `PSSecurityException`, `PipelineStoppedException`, `PipelineClosedException`, `ActionPreferenceStopException`, `ApplicationFailedException`, `NativeCommandExitException`, `ScriptRequiresException`, `HostException`, `RedirectedException`, `RemoteException`, `WildcardPatternException`, `PSSnapInException`, `TypeTableLoadException`, `FormatTableLoadException`. Direct subclasses of `SystemException` that are **not** runtime exceptions include the flow-control family (`FlowControlException`: break, continue, return, exit, terminate), `HaltCommandException`, `ScriptCallDepthException`, `PipelineDepthException`, `RethrowException`, `ParentContainsErrorRecordException`, the "invalid state" exceptions (`InvalidRunspaceStateException`, `InvalidPipelineStateException`, `InvalidPowerShellStateException`, `InvalidRunspacePoolStateException`, `InvalidJobStateException`), `JobFailedException`, `HelpNotFoundException`, `PSConsoleLoadException`, and `AssertException`. +- **`$Error`** (`AppendDollarError`): accepts an `ErrorRecord` or an `Exception`; the list is a global constant `ArrayList`; an object identical to the first element, or an error record whose exception equals the first record's exception, is not added again; the list keeps at most **256** entries (`maxErrorCount` constant; older entries beyond 255 are removed before inserting at index 0), newest first (src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L818-L858). +- **`$?`**: a read-only, all-scope variable whose value is the context's `QuestionMarkVariableValue`; assignments are blocked by the read-only option; the value is set by statement handling (`CheckActionPreference` sets it false; statements set it true after success) and by command completion (for native commands see below) (src/System.Management.Automation/engine/QuestionMarkVariable.cs::QuestionMarkVariable@L9-L42). +- **`$LASTEXITCODE`** and native failure: after a native process exits its exit code is stored in `$LASTEXITCODE`; a non-zero code marks the pipeline's execution as failed (so `$?` is false); only when `$PSNativeCommandUseErrorActionPreference` is true is an error record `ProgramExitedWithNonZeroCode` (`NativeCommandExitException`) written through the normal error path, so `$ErrorActionPreference = 'Stop'` then terminates (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237). +- **Error views** (`$ErrorView`, `ErrorView` enum): `NormalView=0`, `CategoryView=1`, `ConciseView=2`, `DetailedView=3` (src/System.Management.Automation/engine/CommandBase.cs::ErrorView@L312-L325); rendering is format data/code (`UNVERIFIED`, VB-048). +- **Transcripts**: transcript state is `TranscriptionData` (list of transcripts, an optional system-wide transcript, the command being ignored, a helper flag, prompt text `PS>`). `TranscribeCommand` appends the command text to each active transcript: for the first line it can add a header (`**********************`, "Command start time: yyyyMMddHHmmss", `**********************`) when the transcript includes invocation headers, then `PS>` plus text; continuation lines get `>> `; commands that the transcript ignores (helper commands such as `Out-Default` added for transcription) are skipped. `TranscribeResult`, `TranscribeError`, `TranscribeCommandComplete` and `TranscribePipelineComplete` are the other hooks; output is buffered in `OutputToLog` and flushed to the file with the file's detected encoding (src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs::PSHostUserInterface.TranscribeCommand@L418-L460, `TranscriptionOption@L1213-L1300`). + +**Terminating versus non-terminating errors; try/catch/finally/trap ordering (CF-ERR-001).** (1) A *non-terminating* error is an `ErrorRecord` written to the error stream by a command (`WriteError`); it does not interrupt the command or the pipeline, sets the pipeline's failed state and `$?` to false, and is subject to the command's `ErrorAction` (`Ignore`: dropped, not recorded; `SilentlyContinue`: recorded in `$Error`/`-ErrorVariable` but not shown; `Continue`: recorded and written; `Inquire`: ask, then as `Continue` or stop; `Stop`: converted to a terminating `ActionPreferenceStopException`). (2) A *terminating* error is an exception: `ThrowTerminatingError` from a cmdlet (wrapped as `CmdletInvocationException`), `throw`, a runtime error in an expression, or `ActionPreferenceStop`. It stops the current *statement* and, unless handled, the command, and is recorded as the pipeline's first terminating error (later ones are only logged). (3) In **script code every statement is guarded**: when a statement throws a non-flow-control exception the engine consults `$ErrorActionPreference` (default `Continue`): `SilentlyContinue`/`Ignore` resume with the next statement; `Continue` reports the error record and resumes with the next statement — except that an exception produced by a `throw` statement is rethrown when no trap is present; `Stop` rethrows (terminates the enclosing scope and propagates); `Inquire` prompts. (4) **Handler order** for one thrown exception: innermost enclosing `try` is searched first; its `catch` clauses are tried in source order with the first matching type winning (a catch without a type matches any `RuntimeException`; `PipelineStoppedException` is never caught by a type-less catch); `finally` runs after the try/catch block completes, on normal exit, after a caught exception, and while an unhandled exception propagates; if no `catch` in the `try` matches, the exception propagates to the enclosing `try`, then to `trap` handlers of the enclosing blocks (a trap matches by the exception type, trying the inner exception before the outer one), then to the statement-level preference rule of (3). A `trap` runs in a new scope with `$_` set to the error record; `continue` in a trap means *SilentlyContinue*, `break` means *Stop*; otherwise the action is the current preference. (5) Flow control (`break`, `continue`, `return`, `exit`) travels as exceptions that are never caught by `catch` or traps and are rethrown by the statement guard. (6) `$?` is true after a statement that completes without error and false after a statement in which an error (terminating-handled or non-terminating) occurred; the pipeline-level failed state also covers errors written by commands. + +## 5. Data model + +`ErrorRecord` instances are wrapped in `PSObject` with a stream tag when written (`WriteStreamType.Error`) so that merging operators can recognize them. The history list is per runspace (global `$Error`), plus per-command variable lists (`-ErrorVariable`) in pipes. `ExecutionContext` holds `QuestionMarkVariableValue`, `CurrentExceptionBeingHandled`, and the `PropagateExceptionsToEnclosingStatementBlock` flag (used by traps to decide whether an exception should reach outer handlers). + +## 6. Control flow + +```mermaid +flowchart TD + X[exception in statement] --> FC{flow-control?} + FC -- yes --> RT[rethrow] + FC -- no --> TR{enclosing try with matching catch?} + TR -- yes --> CA[run catch with $_ = ErrorRecord, then finally] + TR -- no --> TP{trap in enclosing block?} + TP -- yes --> TH[run trap in new scope; continue or break decide] + TP -- no --> EA{ErrorActionPreference} + EA -- SilentlyContinue or Ignore --> NX[next statement] + EA -- Continue --> RP[report record; next statement unless from throw] + EA -- Stop --> PR[propagate terminating error] + EA -- Inquire --> HQ[ask host] +``` + +## 7. Algorithms + +The statement guard and trap processing are specified in [RUNTIME-OPERATIONS](../02-language/RUNTIME-OPERATIONS.md) §6.1 (ordered steps with source ranges); try/catch dispatch in [02.5-compiler-and-engine](../02-language/02.5-compiler-and-engine.md) §6.4. The cmdlet-level `WriteError` decision list is in [cmdlet-infrastructure](cmdlet-infrastructure.md) §4. This document does not repeat them. + +## 8. Observable behavior [SPEC] + +- In a script, a failing statement prints an error and the script continues (default `Continue`); `$ErrorActionPreference = 'Stop'` makes it terminate; `throw` terminates unless trapped or caught; `-ErrorAction` applies per command and overrides the preference for non-terminating errors from that command. +- `$Error[0]` is the latest error; at most 256 are kept; the same error is not added twice in a row. +- `$?` reflects the last statement; for native programs it is false when the exit code is non-zero, and `$LASTEXITCODE` holds the code. +- `try { } catch { } finally { }` behaves as in CF-ERR-001; `clean` blocks run even when stopped (see [runspace-and-pipeline](runspace-and-pipeline.md)). +- Transcripts capture commands, their output and errors. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Errors in `clean` blocks are written to the error stream (`ReportCleanupError`) and set `$?` false. A second failure during stop is logged, not thrown. `ScriptCallDepthException` is raised by the stack-depth check and is not a runtime exception. Disposal errors become `CmdletInvocationException` warnings. Incomplete-input parse errors are `IncompleteParseException` and do not enter history. Native stderr text is turned into error records (`FromStdErr`) without marking the pipeline as failed (src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968). + +## 10. Concurrency and threading + +`$Error` and `$?` are per runspace and touched only on the pipeline thread. Transcript buffers are locked per transcript (`lock (transcript.OutputToLog)`, `OutputBeingLogged`) and flushed by host code; `PipelineProcessor.RecordFailure` locks around the first-error record. + +## 11. Extension points and customization hooks + +`$ErrorActionPreference`, `-ErrorAction`, `-ErrorVariable`, `$ErrorView`, (no `MaximumErrorCount` identifier exists in this source, searched; the 256 limit is a constant), `trap`, `try/catch/finally`, custom exceptions deriving from `Exception` (wrapped), `ErrorRecord` with `ErrorDetails` for localized messages, `Write-Error`/`$PSCmdlet.WriteError`/`ThrowTerminatingError`, `Start-Transcript` with options (invocation header, path), PowerShell policy for transcription (`PSConfiguration` transcription settings: enable flag, output directory, invocation header). + +## 12. Dependencies + +Inbound: all commands, the compiler's statement guard, hosts (display of errors), remoting (serialization of records). Outbound: `InvocationInfo`/`ScriptPosition` for positions, the debugger (`Break` preference), format code for views, host UI for prompts and transcripts, `MshLog` for health events. + +## 13. Performance notes + +Statement guards add a try/catch frame per statement block; errors build `InvocationInfo` and script stack trace lazily (`ScriptStackTrace` is a property computed on demand, `UNVERIFIED`); the `$Error` list is an `ArrayList` with `Insert(0)` (O(n) per error, bounded by 256). + +## 14. Security considerations + +Error records can leak data: `TargetObject`, `InvocationInfo.Line` and `PositionMessage` can contain secrets typed on the command line; transcripts store command text and output on disk (the file's access control is the only protection); remote error serialization includes script stack traces. Enforcement here: none beyond policy-driven transcription settings. `PSSecurityException` carries blocked-execution reasons for logging. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| logging | 90 | +| app-control | 16 | +| language-mode | 12 | +| constrained-types | 2 | +| securestring | 1 | +| amsi | 1 | +| **total** | 122 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs` | 91 | +| `src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs` | 31 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: audit records for policy-denied operations are produced independently of user-visible errors; transcripts and logs apply the same redaction rules as the host display; error text shown to a remote caller excludes local paths and script text unless policy allows it. + +## 15. Enterprise considerations + +Security first: §14. Observability: `$Error` history, `-ErrorVariable`, transcripts, health events (`MshLog`), `Get-Error` for detailed views, error ids that are stable strings. Supportability: `FullyQualifiedErrorId` and categories allow filtering. Testability: error behavior is well covered by language tests (§16). Resource limits: `$Error` capped at 256 entries; transcripts unbounded. Missing: structured error export schema, correlation ids across commands/remote sessions. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Classes/Scripting.Classes.Exceptions.Tests.ps1` | 3 | 3 | 14 | +| `test/powershell/Language/Scripting/CleanBlockErrorHandling.Tests.ps1` | 3 | 0 | 49 | +| `test/powershell/Language/Scripting/CommandErrorHandling.Tests.ps1` | 1 | 3 | 22 | +| `test/powershell/Language/Scripting/ErrorHistoryId.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/Language/Scripting/ErrorPosition.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Language/Scripting/OutErrorVariable.Tests.ps1` | 3 | 1 | 15 | +| `test/powershell/Language/Scripting/Trap.Tests.ps1` | 1 | 1 | 4 | +| `test/powershell/Language/Scripting/TryCatch.Tests.ps1` | 1 | 10 | 37 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Get-Error.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Error.Tests.ps1` | 1 | 0 | 10 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Write-Error.Tests.ps1` | 1 | 0 | 11 | +| `test/powershell/Modules/Microsoft.Powershell.Host/Start-Transcript.Tests.ps1` | 1 | 0 | 22 | +| `test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1` | 1 | 2 | 10 | +| `test/powershell/engine/Formatting/ErrorView.Tests.ps1` | 1 | 3 | 22 | +| **total (14 files)** | 20 | 23 | 223 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `Host/Start-Transcript`, `Utility/Write-Error`, `Utility/Get-Error`, `Core/Get-Error`, `engine/Basic/NativeCommandErrorHandling`, `Formatting/ErrorView`, `Classes.Exceptions`, `Scripting/Trap`, `Scripting/CommandErrorHandling`, `Scripting/ErrorHistoryId`, `Scripting/TryCatch`, `Scripting/CleanBlockErrorHandling`, `Scripting/ErrorPosition`, `Scripting/OutErrorVariable`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: errors are split into data (`ErrorRecord`) and control (exceptions) so commands can report many problems without aborting. +- `FACT-SRC`: the default for script statements is to continue after errors, a deliberate departure from most languages (and a frequent source of surprises); preferences make it configurable. +- `INFERENCE`: the exception types that do *not* derive from `RuntimeException` (flow control, halt, stack depth) are those that must pass through `catch` clauses untouched. +- `FACT-SRC`: `$Error` de-duplicates the first element to avoid duplicates when an error is reported at several layers. +- `FACT-SRC`: native-command failures are not errors unless the opt-in preference is set. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTERR-01** [REPLICATE] (area: NLD-ERR) Split errors into records (non-terminating, streamed) and exceptions (terminating), with the `ErrorAction` decision list; it defines script behavior. Evidence: src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968. +- **LSN-RTERR-02** [REPLICATE] (area: NLD-ERR) Statement-level guard with continue-by-default and `throw` as the always-terminating form; keep traps with `continue`/`break` semantics. Evidence: src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788. +- **LSN-RTERR-03** [REPLICATE] (area: NLD-ERR) `$Error` as a bounded, newest-first, de-duplicated history; `$?` as a read-only all-scope variable. Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L818-L858. +- **LSN-RTERR-04** [ADAPT] (area: NLD-ERR) Keep `ErrorRecord` fields and the 32 categories but add a schema version, a correlation id and a redaction marker for sensitive fields. Evidence: src/System.Management.Automation/engine/ErrorPackage.cs::ErrorRecord@L988-L1699. +- **LSN-RTERR-05** [ADAPT] (area: NLD-ERR) Make the `$Error` capacity configurable (the source hard-codes 256 and no preference variable controls it). Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L847-L855. +- **LSN-RTERR-06** [REPLICATE] (area: NLD-ERR) Native exit codes in `$LASTEXITCODE` and `$?`, error on non-zero only by preference. Evidence: src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237. +- **LSN-RTERR-07** [REDESIGN] (area: NLD-SEC) Separate the audit stream (policy decisions, denials) from the user error stream so that `-ErrorAction SilentlyContinue` or `Ignore` can never suppress an audit record. DEV: DEV-ERR-180. Evidence: src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-048): `ErrorRecord` internals (fields, `ToString`, serialization, `ScriptStackTrace`), `ErrorDetails`, the error-view rendering code and format data, `Get-Error`. +- `UNVERIFIED` (VB-049): exact semantics of `ThrowTerminatingError` vs `throw` inside traps for each case; the `try`/`finally` behavior on `PipelineStoppedException`; `RethrowException` use; `HaltCommandException` consumers. +- `UNVERIFIED` (VB-050): `ConsoleHost` transcript cmdlets and the policy-driven transcription path (`PSConfiguration` settings), and the `MshLog` health-event mapping. + +Verification: `ExecutionContext.cs` lines 818–858, `QuestionMarkVariable.cs` (whole file), `MshHostUserInterface.cs` lines 418–470 and 1190–1260, `NativeCommandProcessor.cs` lines 1082–1237 (read for T32), `MshCommandRuntime.cs` lines 2858–3010 (read for T33), and the earlier documents' §6.1/§6.4 text were used; enum members were read from source (`ErrorCategory` 32 values counted from source, `ErrorView` 4 values). Exception lists come from `data/types.json` (`APPROX`). Ranges in `T31_T39.csv`. diff --git a/docs-out/03-runtime/eventing.md b/docs-out/03-runtime/eventing.md new file mode 100644 index 0000000..3b6f093 --- /dev/null +++ b/docs-out/03-runtime/eventing.md @@ -0,0 +1,162 @@ +--- +id: RT-EVENTING-001 +title: Eventing and engine events +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/EventManager.cs, src/Microsoft.PowerShell.Commands.Utility/commands/utility/RegisterObjectEventCommand.cs, src/Microsoft.PowerShell.Commands.Utility/commands/utility/NewEventCommand.cs, src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Eventing and engine events + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. Event actions run as jobs ([jobs-and-concurrency](jobs-and-concurrency.md)); the "pulse" that lets the engine run actions while idle is in [runspace-and-pipeline](runspace-and-pipeline.md) §10. + +## 1. Purpose and scope [SPEC] + +The event model lets script react to .NET events and to engine-generated or user-generated events. A *subscriber* names a source (a .NET object and event name, or a *source identifier* string); a *handler* is either a script-block *action* or a delegate; when an event is raised the engine either runs the action, calls the handler, or queues the event for `Get-Event`/`Wait-Event`. This document covers the manager, subscriptions, delivery rules, the action queue and the built-in engine events. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/EventManager.cs::PSEventManager@L22-L316 | abstract class | public contract: `GenerateEvent`, `SubscribeEvent`, `UnsubscribeEvent`, `GetEventSubscribers`, `ReceivedEvents`, `Subscribers` | B | +| src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager@L321-L1546 | class | the in-process manager | B | +| src/System.Management.Automation/engine/EventManager.cs::PSEventSubscriber@L1852-L2078 | class | one subscription | B | +| src/System.Management.Automation/engine/EventManager.cs::PSEventArgs@L2191-L2311 | class | one event instance | B | +| src/System.Management.Automation/engine/EventManager.cs::EventAction@L2437-L2454 | class | a queued (subscriber, event) pair | B | +| src/System.Management.Automation/engine/EventManager.cs::PSEventJob@L2459-L2647 | class | the job that represents an action subscription | B | +| src/System.Management.Automation/engine/EventManager.cs::PSRemoteEventManager | class | forwards events from a remote session | B | +| src/Microsoft.PowerShell.Commands.Utility/commands/utility/RegisterObjectEventCommand.cs::RegisterObjectEventCommand@L13-L72, NewEventCommand.cs | cmdlets | `Register-ObjectEvent`, `New-Event` | B | + +## 3. Responsibilities and non-responsibilities + +Owns: subscriptions, event ids and queues, delivery rules, runtime generation of event handler types, engine events, forwarding hooks. Does not own: the `*-Event` cmdlet surface (utility family), WMI/CIM event sources, remoting transport of forwarded events (Phase 4). + +## 4. Key types and contracts [IMPL] + +- **`PSEventManager`**: `GenerateEvent(sourceIdentifier, sender, args, extraData[, processInCurrentThread, waitForCompletionInCurrentThread])` creates a `PSEventArgs` (ids from a per-manager counter starting at 1) and calls `ProcessNewEvent`; `SubscribeEvent` overloads take either a .NET `source`+`eventName` or just a `sourceIdentifier`, plus `data`, an optional handler delegate, `supportEvent` (hidden subscription), `forwardEvent` and a maximum trigger count; `ReceivedEvents` is the queue of undelivered events. +- **Subscribing to a .NET event** (`ProcessNewSubscriber`): reject source identifiers that start with `PowerShell.` (reserved; error `ReservedIdentifier`); find the event by name (case-insensitive, public instance or static); WinRT types are not supported; if the type has a writable `EnableRaisingEvents` property it is set to true; the event's delegate type must return `void`; an event-handler **class is generated at run time** with `Reflection.Emit` (dynamic assembly `PSEventHandler`, module `PSGenericEventModule`) matching the delegate signature, cached by `"|"`; a delegate of the generated class is added to the event; when the event fires it calls `GenerateEvent` with the sender, the event arguments and the subscriber's data (src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewSubscriber@L645-L796). +- **Engine events** (`PSEngineEvent`): source identifiers `PowerShell.Exiting` and `PowerShell.OnIdle` (public) and `PowerShell.OnScriptBlockInvoke`, `PowerShell.GetCommandInfoParameterMetadata` (internal). A subscription with no `source` whose identifier is in the engine-event set is an engine subscription (`OnIdle` starts a timer, `UNVERIFIED` details). +- **Delivery** (`ProcessNewEventImplementation`): for each subscriber of the event's source identifier (in subscription order): set the event's `ForwardEvent` flag from the subscriber; if the subscriber has an **action**, enqueue an `EventAction` (captured); else if it has a **handler delegate**, either enqueue it (when it must run in the execution thread) or call it in the current thread after the loop (captured); else remember it as "without action". After the loop, if no subscriber captured the event: if it is a forwarded event, raise the forward event; otherwise **add it to `ReceivedEvents`** (under its sync lock); subscribers without action or handler that were auto-unregister are then unregistered. Handlers run in the current thread are followed by `AutoUnregisterEventIfNecessary` (src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewEventImplementation@L957-L1018). +- **Action queue and execution**: actions are added to a queue under its lock, then the engine is **pulsed** (the runspace's `Pulse` runs an empty pipeline when idle). `ProcessPendingActions` returns at once if the queue is empty or an action is already executing; otherwise, under the processing lock, it dequeues and runs up to `throttleLimit × throttleChecks` actions per call (limit 1, the check counter increases while the queue stays non-empty and resets after progress); in `finally`, if actions remain, it schedules a re-pulse after a 100 ms sleep on the thread pool. `InvokeAction` switches the engine session state to the action script block's session state, sets the default runspace, invokes the action (`Sender.Action.Invoke(sender, args)`) or the delegate, and restores state; a `PipelineStoppedException` from the action re-queues the action; waiting event generators (`EventProcessed` handle) are released when the action completes (src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessPendingActionsImpl@L1072-L1137, `InvokeAction@L1222-L1281`). +- **`PSEventJob`** represents an action subscription as a job (`Get-Job` shows it); `RegisterJob` associates the subscriber with it; unsubscribing stops it. + +## 5. Data model + +Manager state: `_eventSubscribers` (subscriber → generated delegate), `_engineEventSubscribers` (engine event name → subscribers, case-insensitive), `_actionQueue`, the next subscription id, throttle counters, the dynamic assembly/module and a static cache of generated handler types, timer state for `OnIdle`. `PSEventSubscriber`: subscription id, source object, event name, source identifier, action, handler delegate, `SupportEvent`, `ForwardEvent`, `AutoUnregister` (max trigger count), `Data`, job. `PSEventArgs`: event identifier, source identifier, sender, source args, message data (`extraData`), time generated, forward flag, `EventProcessed` wait handle. + +## 6. Control flow + +```mermaid +flowchart TD + SRC[.NET event or New-Event] --> GE[GenerateEvent] + GE --> PN[ProcessNewEvent] + PN --> LOOP[for each subscriber of source identifier] + LOOP -->|has Action| Q[enqueue EventAction and pulse engine] + LOOP -->|has handler delegate| H{runs in execution thread?} + H -->|yes| Q + H -->|no| CT[call handler in current thread] + LOOP -->|neither| RW[remember subscriber] + Q --> PA[ProcessPendingActions when engine idle] + PA --> IA[InvokeAction in subscriber session state] + LOOP -->|nobody captured| RE[ReceivedEvents or forward event] +``` + +## 7. Algorithms + +Generated handler types (one per (source type, event) pair) bind arbitrary delegate signatures to the single `GenerateEvent` call, avoiding reflection per event. Throttling is described in §4. Unsubscribe removes the delegate from the source event and the subscriber from the tables (`UNVERIFIED` details). + +## 8. Observable behavior [SPEC] + +- `Register-ObjectEvent` returns a subscription (and a job when an action is given); events with an action run the action in the subscriber's session state when the engine is idle; events without action are queued for `Get-Event`/`Wait-Event`. +- Events are delivered one action at a time (no overlapping actions); events raised while an action runs wait. +- `New-Event` raises a user event with a source identifier. +- Identifiers beginning with `PowerShell.` are reserved for the engine. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Unknown event name → `CouldNotFindEvent`; non-void delegate → `NonVoidDelegateNotSupported`; WinRT sources unsupported; a throwing action does not unsubscribe (errors surface through the event job, `UNVERIFIED`); a `PipelineStoppedException` re-queues the action; disposal clears subscribers; action processing is skipped if called re-entrantly from within an action. + +## 10. Concurrency and threading + +Events can be raised on any thread (a .NET event source); queue operations lock the queue; processing is serialized by `_actionProcessingLock`; `IsExecutingEventAction` prevents re-entrancy; synchronous generation can wait on the action's `EventProcessed` handle; the engine pulse uses the runspace's pulse pipeline when no pipeline runs. Actions never run concurrently with a running pipeline of the same runspace (they run when the runspace is idle or at pipeline boundaries via `ProcessPendingActions` in the pipeline's `finally`). + +## 11. Extension points and customization hooks + +`Register-ObjectEvent`, `Register-EngineEvent`, `New-Event`, `Get-Event`, `Wait-Event`, `Remove-Event`, `Unregister-Event`, `Get-EventSubscriber`; `-Action`, `-MessageData`, `-SupportEvent`, `-Forward`, `-MaxTriggerCount`; host code can subscribe through `Runspace.Events` (`PSEventManager`); `PSEngineEvent.OnIdle` and `Exiting`. + +## 12. Dependencies + +Inbound: event cmdlets, module `OnRemove`/exit handling (`Exiting` event; `UNVERIFIED`), hosts. Outbound: `LocalRunspace.Pulse` (engine idle processing), session state of the action script block, `Reflection.Emit` (dynamic types), jobs (`PSEventJob`), `ExecutionContext`. + +## 13. Performance notes + +Handler classes are generated once per (type, event) and cached; each event costs a queue operation and an engine pulse; throttle limit 1 per pulse with a 100 ms re-pulse bounds CPU use but adds latency under load; `ReceivedEvents` is unbounded. + +## 14. Security considerations + +Event actions are script blocks that run later in the subscriber's session state, which can be a more privileged state than the code that raises the event; the source of an event can be any .NET object the script can reach. Dynamic code generation (`Reflection.Emit`) creates types at run time (not available under some application-control policies; `UNVERIFIED`). Forwarded events carry data to another session (remoting) and must be treated as untrusted on the receiving side. The reserved `PowerShell.` prefix protects engine events from user spoofing at subscription time (not at `New-Event`, `UNVERIFIED`). + +### Security surface + + +No scanner seed hit under the prefixes of this subsystem (`UNVERIFIED`: absence of a hit is not proof of absence of a check). + + +What the new shell must guarantee here: an action runs with the language mode and permissions of its subscriber, never of the raiser; engine event identifiers can be generated only by the engine; event queues are bounded and rate-limited; forwarded events are tagged with origin and are not trusted. + +## 15. Enterprise considerations + +Security first: §14. Observability: `Get-EventSubscriber`, `Get-Event`, event jobs, event ids and time stamps. Supportability: source identifiers give stable names for monitoring. Testability: events are driven through cmdlets and `GenerateEvent`. Resource limits: `ReceivedEvents` unbounded; one action at a time. Missing: bounded queues, back-pressure, delivery guarantees, audit of subscriptions. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Modules/Microsoft.PowerShell.Management/New-EventLog.Tests.ps1` | 1 | 0 | 5 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Remove-EventLog.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Eventing.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Event.Tests.ps1` | 1 | 2 | 4 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-EventSubscriber.Tests.ps1` | 1 | 2 | 3 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/New-Event.Tests.ps1` | 1 | 2 | 3 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Register-EngineEvent.Tests.ps1` | 1 | 1 | 1 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Register-ObjectEvent.Tests.ps1` | 1 | 2 | 3 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Remove-Event.Tests.ps1` | 1 | 1 | 4 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Wait-Event.Tests.ps1` | 1 | 1 | 1 | +| **total (10 files)** | 10 | 11 | 29 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `Register-EngineEvent`, `New-Event`, `Register-ObjectEvent`, `Get-EventSubscriber`, `Remove-Event`, `Get-Event.Tests`, `Utility/Eventing`, `Wait-Event`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: actions run only when the runspace is idle (no overlap with user code), which avoids re-entrancy in a single-threaded session at the price of delivery latency. +- `FACT-SRC`: handler types are generated at run time to support any delegate signature. +- `INFERENCE`: the throttle of one action per pulse protects interactive use. +- `FACT-SRC`: events nobody handles accumulate in `ReceivedEvents` rather than being dropped. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTEVT-01** [REPLICATE] (area: NLD-RUN) Source-identifier based events with queue, actions, auto-unregister and reserved engine identifiers. Evidence: src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewEventImplementation@L957-L1018. +- **LSN-RTEVT-02** [REPLICATE] (area: NLD-RUN) Run actions serially at idle points in the subscriber's session state. Evidence: src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.InvokeAction@L1222-L1281. +- **LSN-RTEVT-03** [ADAPT] (area: NLD-RUN) Replace `Reflection.Emit` handler generation with expression-tree or `Delegate.CreateDelegate` adapters (or `DynamicInvoke` fallback) so the feature works where dynamic assemblies are blocked. Evidence: src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewSubscriber@L645-L796. +- **LSN-RTEVT-04** [REDESIGN] (area: NLD-SEC) Bound the received-event queue and action queue with policy-defined limits and drop/oldest rules. DEV: DEV-EVT-198. Evidence: src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessPendingActionsImpl@L1072-L1137. +- **LSN-RTEVT-05** [AVOID] (area: NLD-RUN) Do not port forwarded events (`PSRemoteEventManager`) or the WMI/CIM event sources in the first version. DEV: DEV-EVT-199. Evidence: src/System.Management.Automation/engine/EventManager.cs::PSRemoteEventManager@L1551-L1811. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-056): `OnIdle` timer logic, `UnsubscribeEvent`, `GenerateEventHandler` (IL emission), `PSEventSubscriber` members, `PSEventJob`, `PSRemoteEventManager`, `Register-ObjectEvent` parameter handling, `PSEventArgsCollection`. +- `UNVERIFIED` (VB-057): where the `Exiting` event is raised and who consumes `OnScriptBlockInvoke`. + +Verification: `EventManager.cs` lines 22–140, 326–392 (selected), 510–572, 645–760, 957–1137, 1222–1295 and 1821–1850 were read; the rest is from signatures. Ranges in `T31_T39.csv`. diff --git a/docs-out/03-runtime/extended-type-system.md b/docs-out/03-runtime/extended-type-system.md new file mode 100644 index 0000000..e81799a --- /dev/null +++ b/docs-out/03-runtime/extended-type-system.md @@ -0,0 +1,180 @@ +--- +id: RT-ETS-001 +title: Extended type system +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-ETS-001] +sources: [src/System.Management.Automation/engine/MshObject.cs, src/System.Management.Automation/engine/MshMemberInfo.cs, src/System.Management.Automation/engine/CoreAdapter.cs, src/System.Management.Automation/engine/TypeTable.cs, src/System.Management.Automation/engine/ExtraAdapter.cs, src/System.Management.Automation/engine/ThirdPartyAdapter.cs, src/System.Management.Automation/engine/ManagementObjectAdapter.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/types.json, data/enums.json, data/format-and-types-xml.json, data/tests.json, data/enforcement-points.json] +--- + +# Extended type system + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Conversion rules are in [CONVERSION-RULES](../02-language/CONVERSION-RULES.md); how compiled code reads members through binders in [BINDER-CATALOG](../02-language/BINDER-CATALOG.md) and [RUNTIME-OPERATIONS](../02-language/RUNTIME-OPERATIONS.md); the minimal ETS the core needs is called out in §18. + +## 1. Purpose and scope [SPEC] + +The extended type system (ETS) gives every value a uniform *member view*: properties, methods, events and sets of them, regardless of whether the value is a .NET object, XML node, database row, COM object, a deserialized object or a script-made custom object. `PSObject` wraps a value; *adapters* expose the value's native members; the *type table* adds script-level members by type name; instance members can be added to a single object (`Add-Member`). Scripts, formatting, serialization and `Select-Object` all work through this view. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/MshObject.cs::PSObject@L47-L2430 | class | wrapper; member collections; adapter selection; type names | B | +| src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfo@L173-L299 | abstract class | base of all member kinds | B | +| src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152 | class | the merged member view used by `PSObject.Members` | B | +| src/System.Management.Automation/engine/MshMemberInfo.cs::CollectionEntry@L4367-L4417 | class | one source of members (type table, adapter, base) | B | +| src/System.Management.Automation/engine/CoreAdapter.cs::Adapter@L38-L2049 | abstract class | adapter contract (get member, property get/set, method invoke, definitions) | B | +| src/System.Management.Automation/engine/CoreAdapter.cs::DotNetAdapter@L2616-L4655 | class | reflection-based adapter with per-type caches | B | +| src/System.Management.Automation/engine/TypeTable.cs::TypeTable@L2764-L4778 | class | members, converters, adapters, serialization data by type name | B | +| src/System.Management.Automation/engine/ExtraAdapter.cs, ManagementObjectAdapter.cs, ThirdPartyAdapter.cs | classes | XML/DataRow/DirectoryEntry/WMI/CIM adapters; `PSPropertyAdapter` bridge | B | +| src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs (+2 siblings) | generated partial `TypeTable` | the built-in type data compiled into code (instead of loading `types.ps1xml`) | B | + +## 3. Responsibilities and non-responsibilities + +Owns: member lookup and enumeration, adapter choice, type-name hierarchy, instance and type-table members, `PSObject` identity and base-object unwrapping, type-data update. Does not own: operator semantics and conversion (T28), method overload resolution at call sites (binders, T26), formatting (T36), serialization format (Phase 4 remoting). + +## 4. Key types and contracts [IMPL] + +- **`PSObject`** holds the wrapped value (`BaseObject`), instance members, optional type-name list (`TypeNames`/`pstypenames`), the adapter set, deserialization fields (`AdaptedMembers`, `ClrMembers`, `IsDeserialized`) and the `WriteStream` tag used by the stream-merging code. `AsPSObject` wraps a value (or returns it if already wrapped); `Base(obj)` unwraps. A value of type `PSCustomObject`-style "empty base" has `ImmediateBaseObjectIsEmpty`. +- **Member kinds** (`PSMemberTypes` flags): `AliasProperty=1`, `CodeProperty=2`, `Property=4`, `NoteProperty=8`, `ScriptProperty=16`, `PropertySet=32`, `Method=64`, `CodeMethod=128`, `ScriptMethod=256`, `ParameterizedProperty=512`, `MemberSet=1024`, `Event=2048` (src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberTypes@L33-L118). Classes: `PSAliasProperty`, `PSCodeProperty`, `PSProperty` (adapted `PSAdaptedProperty`), `PSNoteProperty` (`PSVariableProperty`), `PSScriptProperty`, `PSMethod`, `PSCodeMethod`, `PSScriptMethod`, `PSParameterizedProperty`, `PSMemberSet`, `PSPropertySet`, `PSEvent`, `PSDynamicMember`. +- **Views** (`PSMemberViewTypes` flags): `Extended=1`, `Adapted=2`, `Base=4`, `All`. +- **Adapter contract** (`Adapter`): `GetMember(obj, name)`, `GetFirstMemberOrDefault`, `GetMembers(obj)`, `GetTypeNameHierarchy`, property operations (`PropertyGet/Set`, `IsGettable/IsSettable`, `PropertyType`, `PropertyAttributes`), method operations (`MethodInvoke`, `MethodDefinitions`), parameterized-property operations, and `CanSiteBinderOptimize`. An `AdapterSet` pairs the *original* adapter with an optional `.NET` adapter used for the *base* view. +- **Adapter selection** (`GetMappedAdapter(obj, typeTable)`): (1) an adapter registered in the type table for the object's type **by exact full type name** (no inheritance; source comment marks this as a known limitation); (2) a cached per-type result; (3) the built-in mappers in this order: `PSMemberSet` → member-set adapter; `PSObject` → `PSObject` adapter; `CimInstance` → CIM adapter; on Windows `ManagementClass`, `ManagementBaseObject`, `DirectoryEntry`; `DataRowView`; `DataRow`; `XmlNode`; (4) otherwise a COM object (Windows) gets a COM adapter built from its type info (or the .NET adapter with COM type name), and everything else gets the .NET instance adapter. The result is cached per type (src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMappedAdapter@L460-L538). +- **Type table**: `GetMembers(typeNames)` builds (and caches by the joined type-name key) a merged member collection: type names are processed from the **last (least specific) to the first (most specific)**; a member of a more specific type replaces a same-named member of a less specific type; if both are member sets and the new one has `InheritMembers`, their members are merged instead (new members added, same-named members replaced). Converters and adapters are looked up by name (src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041). +- **Script members**: a `PSScriptProperty` created from text (type data) records a `PSLanguageMode` and applies it to its getter and setter script blocks when they are created, so the script runs under the language mode in force when the type data was loaded; getter and setter run with the object as `$this` and are marked step-through for the debugger (src/System.Management.Automation/engine/MshMemberInfo.cs::PSScriptProperty@L1537-L1905). + +**Member resolution order (CF-ETS-001).** Looking up member *name* on a value `v` (wrapped as a `PSObject`) proceeds in this order and returns the first hit: (1) the reserved names `psbase`, `psadapted`, `psextended`, `psobject` and `pstypenames`, generated on demand; (2) **instance members** added to this object (for example by `Add-Member`); (3) **type-table (extended) members** merged over the object's type-name hierarchy, most specific type name winning; (4) **adapted members** from the adapter chosen for the object; (5) **base members**: the .NET members of the wrapped object, only when the chosen adapter is not already the .NET adapter. Member-name matching in the .NET adapter is case-insensitive (its binding flags include `IgnoreCase`). Type-table collection entries are marked "replicate and clone when returning" (a copy bound to the current object is handed out), adapted and base entries are not; the enumeration order across sources is `UNVERIFIED` (VB-038). Member lookup on a `PSMemberSet` owner searches the set's own members first, then the same collections. + +## 5. Data model + +`PSObject` fields: base object, `InstanceMembers` (lazily created collection), `_typeNames` (`ConsolidatedString` with an interned key used for type-table caching), `_adapterSet`, `HasGeneratedReservedMembers`, `ToStringFromDeserialization`, `TokenText`, `WriteStream`. `TypeTable`: maps type name → members (`_extendedMembers`), type name → converter, type name → adapter, type name → serialization settings (`GetSpecificProperties`), and a consolidated-members cache. Static caches in `DotNetAdapter`: per-`Type` property, method and event cache tables for instance and static views (`s_instancePropertyCacheTable`, `s_staticMethodCacheTable`, …) and `s_adapterMapping` (`Type` → `AdapterSet`). + +## 6. Control flow + +```mermaid +flowchart TD + Q[member name on object] --> R{reserved name?} + R -- yes --> RS[generate reserved member set] + R -- no --> I[instance members] + I -- miss --> TT[type table members over type-name hierarchy] + TT -- miss --> AD[adapter members] + AD -- miss --> BS[base .NET members] + BS -- miss --> NF[not found] +``` + +## 7. Algorithms + +Adapter selection and type-table merge are given in §4. Type-name hierarchy for a .NET object comes from the adapter (`GetTypeNameHierarchy`): for the .NET adapter the runtime type, its base types and interfaces as arranged by the adapter (`UNVERIFIED` exact order, VB-038); instance `TypeNames` can be replaced per object (`pstypenames`). The merged type-table collection is cached by the joined type-name key, so changing type data clears it (`Update` in `TypeTable`). + +## 8. Observable behavior [SPEC] + +- `Get-Member` lists members of all views (`-View Extended|Adapted|Base|All`); `$o.psobject.Properties` is the property view; `$o.pstypenames` shows the type-name list. +- `Add-Member` adds an instance member; adding a name that the type data already defines is an error (`AlreadyPresentInTypesXml`); reserved names cannot be added (`PSObjectMembersMembersAddReservedName`); adding a plain `Property` or `Method` is not allowed (`CannotAddMethodOrProperty`). +- `Update-TypeData` / `Remove-TypeData` change type-table members. +- A deserialized object keeps its adapted/CLR members as data and loses behavior. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`ExtendedTypeSystemException` with ids such as `CannotAddMethodOrProperty`, `CannotChangeReservedMember`, `RemoveMemberFromStaticMemberSet`, `AlreadyPresentInTypesXml`; `GetValueException` / `SetValueException` when a getter or setter is missing or throws (for example `GetWithoutGetterFromScriptPropertyValue`); adapter exceptions during `IsGettable` etc. are converted to ETS exceptions with `CatchFrom…` ids. Case-insensitive lookup in the .NET adapter can make two members with names differing only by case ambiguous (`UNVERIFIED` handling). + +## 10. Concurrency and threading + +Static caches (`s_adapterMapping`, the `DotNetAdapter` per-type tables) are shared by all runspaces; adapter mapping takes a lock for the mapper list and uses concurrent dictionaries; `TypeTable` consolidated members are held in a concurrent cache. A `PSObject` itself is not thread-safe. + +## 11. Extension points and customization hooks + +`Update-TypeData` (type, member kind, serialization settings), type `*.ps1xml` files, `PSPropertyAdapter` (third-party property adapters registered by type name through the type table), `Add-Member` instance members (note, script property/method, alias, code members, member sets), `ETS` converters (`PSTypeConverter`, `TypeConverter` registered in type data), `DefaultDisplayPropertySet` and `PSStandardMembers` for display and serialization defaults (`GetPSStandardMember`). + +## 12. Dependencies + +Inbound: every cmdlet and the runtime (`PSObject.AsPSObject` at pipe boundaries), the binders (member access), formatting, serialization, `Select-Object`, `Where-Object`, `Sort-Object`. Outbound: `LanguagePrimitives` (enumeration, conversion), `ExecutionContext` (the type table; `GetTypeTable` falls back to the thread's context), reflection, script blocks for script members. + +## 13. Performance notes + +Per-type reflection caches (properties, methods with `MethodInformation` arrays, events) avoid repeated reflection; the adapter mapping is cached per `Type`; the consolidated type-table members are cached per type-name key; `CanSiteBinderOptimize` lets call-site binders skip the generic path for the .NET adapter. Member collections for instance members are created lazily. Wrapping every pipeline value in `PSObject` allocates; `ImmediateBaseObjectIsEmpty` and `AsPSObject` reuse limit it. + +## 14. Security considerations + +ETS is a policy surface: (a) script properties and script methods from type data run script, under the language mode captured at load; (b) the type table is a global mutation point — `Update-TypeData` can change how values behave for later code, so it must be restricted in constrained sessions (`UNVERIFIED` where enforced, Phase 5); (c) adapters decide which native members are visible — the `.NET` adapter exposes public members of any type that the process can load, which is why constrained language restricts method calls and property sets on non-core types at the binder level (not here); (d) deserialized objects must not regain behavior (`IsDeserialized` keeps data-only members). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 10 | +| constrained-types | 1 | +| **total** | 11 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/MshMemberInfo.cs` | 8 | +| `src/System.Management.Automation/engine/TypeTable.cs` | 2 | +| `src/System.Management.Automation/engine/MshObject.cs` | 1 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: type-data mutation is a policy-controlled operation; script members are bound to the language mode of their definer; member visibility for a value can be restricted per session without changing the value; deserialized data never resolves to live adapters. + +## 15. Enterprise considerations + +Security first: §14. Observability: a member-resolution tracer (`PSObject.MemberResolution`) records each lookup step ("Found PSObject instance member", "present in type table", "Adapted member", "DotNet member"). Supportability: `Get-Member` plus the tracer explain where a member comes from. Testability: ETS is exercised through the language, plus xUnit tests on `PSObject`. Resource limits: unbounded caches keyed by type; no eviction (`INFERENCE`). Missing: scoped (per-runspace) adapter registration for non-type-table adapters (the mapping cache is static). + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Scripting/Adapter.Tests.ps1` | 2 | 0 | 5 | +| `test/powershell/Language/Scripting/HashtableToPSCustomObjectConversion.Tests.ps1` | 2 | 0 | 11 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Add-Member.Tests.ps1` | 2 | 0 | 25 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Member.Tests.ps1` | 2 | 3 | 15 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Remove-TypeData.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Update-TypeData.Tests.ps1` | 1 | 1 | 18 | +| `test/powershell/engine/ETS/Adapter.Tests.ps1` | 4 | 7 | 26 | +| `test/powershell/engine/ETS/CimAdapter.Tests.ps1` | 1 | 0 | 15 | +| `test/powershell/engine/ETS/TypeTable.Tests.ps1` | 1 | 0 | 4 | +| **total (9 files)** | 16 | 11 | 122 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_PSObject.cs` | 8 | 0 | `PSObjectTests` | + +Matching is by path keyword (`APPROX`): `engine/ETS/`, `Scripting/Adapter.Tests`, `Utility/Update-TypeData`, `Utility/Remove-TypeData`, `Utility/Add-Member`, `Utility/Get-Member`, `HashtableToPSCustomObjectConversion`, `test_PSObject`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: layered member view (instance → type table → adapter → base) lets scripts add or override behavior without changing the runtime type. +- `FACT-SRC`: adapter lookup by exact type name is documented in source as a stop-gap (comment says it should honour type hierarchy). +- `INFERENCE`: wrapping values in `PSObject` only at boundaries (not on every operation) balances the cost of the dual view. +- `FACT-SRC`: the built-in type data is compiled into C# (`TypeTable_Types_Ps1Xml.cs`) so start-up does not parse XML. +- `FACT-SRC`: reserved names are generated lazily the first time any reserved member is asked for. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTETS-01** [REPLICATE] (area: NLD-TYP) Keep the member resolution order instance → type data → adapter → base with the five reserved names; scripts depend on it. Evidence: src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152. +- **LSN-RTETS-02** [REPLICATE] (area: NLD-TYP) Merge type data from the least to the most specific type name, with member-set inheritance. Evidence: src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041. +- **LSN-RTETS-03** [ADAPT] (area: NLD-TYP) Select adapters by type with inheritance (nearest registered base type wins) instead of exact full-name match, as the source comment itself proposes. Evidence: src/System.Management.Automation/engine/TypeTable.cs::TypeTable.GetTypeAdapter@L4064-L4099. +- **LSN-RTETS-04** [ADAPT] (area: NLD-TYP) Make adapter and type-table registries per-runspace (or per-session-state) objects rather than static caches, so one session cannot change another's member view. Evidence: src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMappedAdapter@L460-L538. +- **LSN-RTETS-05** [REPLICATE] (area: NLD-SEC) Bind script members to the definer's language mode and keep deserialized objects data-only. Evidence: src/System.Management.Automation/engine/MshMemberInfo.cs::PSScriptProperty@L1537-L1905. +- **LSN-RTETS-06** [AVOID] (area: NLD-TYP) Do not carry the WMI, DirectoryEntry, DataRow/DataRowView and COM adapters into the first version; keep .NET, XML, `PSObject`, member-set and the third-party adapter bridge. DEV: DEV-ETS-150. Evidence: src/System.Management.Automation/engine/MshObject.cs::PSObject.MappedInternalAdapterSet@L433-L454. +- **LSN-RTETS-07** [REDESIGN] (area: NLD-TYP) Compile the built-in type data to a table at build time (as PowerShell does) but also publish it as a versioned data file, so tooling and a clean-room implementation share one source. DEV: DEV-ETS-151. Evidence: src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs::TypeTable@L12-L9138. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-038): `DotNetAdapter` member discovery (overload tables, property caches, hidden members, case-collision handling), type-name hierarchy order, `PSMethod` invocation path, `PSObject` members not read (formatting-related `ToString`, comparison, `Copy`, serialization helpers), `TypeTable.Update` and types.ps1xml loading. +- `UNVERIFIED` (VB-039): `PSMemberSet`/`PSPropertySet` semantics beyond lookup; `PSEvent`; `PSDynamicMember`. + +Verification: `MshObject.cs` lines 245–330, 426–545, 1938–2116, `MshMemberInfo.cs` lines 4503–4560, 4799–4860 and the member-type enums and class list, `TypeTable.cs` lines 3965–4110, `CoreAdapter.cs` declarations (lines 38–330, 2616–2700) were read. Remaining content is from signatures (tier B). Ranges in `T31_T39.csv`. diff --git a/docs-out/03-runtime/formatting-and-output.md b/docs-out/03-runtime/formatting-and-output.md new file mode 100644 index 0000000..092a2d4 --- /dev/null +++ b/docs-out/03-runtime/formatting-and-output.md @@ -0,0 +1,203 @@ +--- +id: RT-FORMAT-001 +title: Formatting and output +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs, src/System.Management.Automation/FormatAndOutput/format-default/format-default.cs, src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs, src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs, src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs, src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs, src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/FormatAndOutput/_folder.md] +generated_from: [data/types.json, data/format-and-types-xml.json, data/tests.json, data/enforcement-points.json] +--- + +# Formatting and output + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. The run-time command that ends every interactive pipeline is `Out-Default` (see [runspace-and-pipeline](runspace-and-pipeline.md) §14: transcription appends it). Member access used by views: [extended-type-system](extended-type-system.md). Error rendering: [errors-and-diagnostics](errors-and-diagnostics.md). + +## 1. Purpose and scope [SPEC] + +Turns objects into text for a host. Objects that reach `Out-Default` are given to a hidden formatting step (`Format-Default`) which picks a *view* (table, list, wide, or custom/"complex") from a *display database* built from format data (`*.format.ps1xml`) and produces *formatting records*; a writer then lays the records out as lines on a `LineOutput` (console, string, file). This document covers the pipeline of format/out commands, view selection, the record protocol, layout caching and the output sinks. Colors and ANSI (`PSStyle`) are named but not analysed. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs::OutDefaultCommand@L44-L163 | cmdlet | `Out-Default`: sends objects to the console writer; captures `-OutVariable` | B | +| src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs::OutHostCommand@L169-L211 | cmdlet | `Out-Host` | B | +| src/System.Management.Automation/FormatAndOutput/format-default/format-default.cs::FormatDefaultCommand@L14-L38 | cmdlet | `Format-Default`: shape undefined (let the database decide) | B | +| src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs::FrontEndCommandBase@L147-L270 | abstract cmdlet | thin cmdlet shell delegating to an *implementation* object | B | +| src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs::CommandWrapper@L39-L140 | class | runs a cmdlet as a private sub-pipeline step-by-step | B | +| src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs::OutputManagerInner@L19-L134 | class | implementation of Out-Default/Out-Host: owns the sub-pipeline manager | B | +| src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs::SubPipelineManager@L141-L326 | class | creates the hidden `Out-LineOutput` sub-pipeline | B | +| src/System.Management.Automation/FormatAndOutput/out-textInterface/OutTextInterface.cs::OutLineOutputCommand@L18-L91 | cmdlet | `Out-LineOutput`: consumes formatting records, writes lines | B | +| src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394 | class | formatting-record consumer: state machine, caching, group handling | B | +| src/System.Management.Automation/FormatAndOutput/common/BaseFormattingCommand.cs::InnerFormatShapeCommand@L44-L543 | class | producer: selects a view and emits formatting records | B | +| src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs::FormatViewManager@L69-L460 | class | view selection | B | +| src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs::DisplayDataQuery@L244-L672 | class | queries the database by shape, type names and view name | B | +| src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataManager.cs::TypeInfoDataBaseManager@L20-L569 | class | loads and holds the database | B | +| src/System.Management.Automation/FormatAndOutput/common/TableWriter.cs, ListWriter.cs, ComplexWriter.cs | classes | line layout per shape | B | +| src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs::LineOutput@L215-L304 | abstract class | the output sink contract | B | +| src/System.Management.Automation/FormatAndOutput/out-console/ConsoleLineOutput.cs::ConsoleLineOutput@L83-L548 | class | console sink (paging, wrapping) | B | +| src/System.Management.Automation/FormatAndOutput/DefaultFormatters/*.cs | generated classes | built-in format data compiled into code | B | +| src/System.Management.Automation/FormatAndOutput/common/PSStyle.cs::PSStyle@L43-L913 | class | ANSI styles and output-rendering mode | B | + +## 3. Responsibilities and non-responsibilities + +Owns: view selection, formatting-record protocol, layout and column width logic, sink contract. Does not own: object member access (ETS), host UI primitives (`PSHostUserInterface`, Phase 4), the cmdlets `Format-Table`, `Format-List`, `Format-Wide`, `Format-Custom`, `Out-File`, `Out-String` (command-module family `Microsoft.PowerShell.Utility`; they reuse `InnerFormatShapeCommand` / `OutCommandInner`). + +## 4. Key types and contracts [IMPL] + +- **`FrontEndCommandBase`** (base of `Out-Default`, `Out-Host`, `Format-*`, `Out-LineOutput`) holds one `ImplementationCommandBase` and wires three callbacks (outer cmdlet, read input object, write object). All real work is in the implementation, so the same logic runs as a cmdlet or inside a sub-pipeline. +- **`CommandWrapper`** builds a one-command `PipelineProcessor` over a `CmdletInfo`/`CommandProcessor` pair and drives it with `Process(o)` (calls `Step`) and `ShutDown()` (calls `SynchronousExecuteEnumerate`), so formatting commands can call each other without a parser (src/System.Management.Automation/FormatAndOutput/common/BaseCommand.cs::CommandWrapper@L39-L140). +- **Out-Default flow**: `BeginProcessing` creates a console `LineOutput`, sets `MergeUnclaimedPreviousErrorResults = true` on its runtime (so earlier commands' unclaimed error output is routed to it), optionally starts transcribe-only mode (`-Transcript`), and, when `-OutVariable` is bound, collects non-error, non-formatting objects; `ProcessRecord` forwards each input object to `OutputManagerInner`, which lazily creates a `SubPipelineManager` that runs the hidden `Out-LineOutput` cmdlet with the line sink as a named parameter (src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs::OutDefaultCommand@L44-L163, src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs::SubPipelineManager@L141-L326). The manager has one *default* handler (`out-lineoutput`) and a list of per-type handlers (empty in this version). +- **Raw object vs formatting record**: in `OutCommandInner.ProcessRecord`, an object is first tried as a formatting record (`FormatStartData`, `GroupStartData`, `FormatEntryData`, `GroupEndData`, `FormatEndData`); if it is not one, it is sent through a private `format-default` command (`ApplyFormatting`) and each result is processed as a record (src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394). +- **Record state machine** (`NeedsPreprocessing`): states `Reset` → (`FormatStartData`) `Formatting` → (`GroupStartData`) `InsideGroup` → entries → (`GroupEndData`) → `Formatting` → (`FormatEndData`) `Reset`. An entry that is "out of band" is allowed anywhere. A new `FormatStartData` while inside a group ends the current formatting first (flush) and begins again. Wrong order raises a formatting error. + +- **View selection** (`FormatViewManager.Initialize`): inputs are the object's type-name list (`InternalTypeNames`), a requested shape (`Undefined` for `Format-Default`), and command-line parameters. (1) shape undefined → `GetDefaultView` over the database; (2) a view found → use it; (3) not found → `SelectViewGeneratorFromProperties`; with explicit properties → property view of the requested shape; with an explicit `-View` name → that named view of the shape, else `ProcessUnknownViewName` raises an error; otherwise → the first applicable view of the shape; none → property view (src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs::FormatViewManager.Initialize@L91-L204). A view matches by *applies-to* lists of type names and type groups; the database is walked in order, skipping out-of-band list/complex views and views of other shapes; the first *perfect match* of the object's type names wins, otherwise the *best match* (`TypeMatch.BestMatch`); if none matches and the type names carry the deserialization prefix of the type names, the search repeats without the prefix (src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs::DisplayDataQuery@L244-L672). `UNVERIFIED` (VB-040): the exact `TypeMatch.PerfectMatch`/`BestMatch` ranking. +- **Property view fallback** (no view in the database): the shape comes from the database's shape directives for the type, else from the count of the object's *default display property set* (`PSStandardMembers.DefaultDisplayPropertySet`), falling back to *all* properties when no default set exists: **table when the count is at or below the database's `PropertyCountForTable` threshold, else list** (src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs::DisplayDataQuery.GetShapeFromPropertyCount@L333-L339). The threshold value is data (`UNVERIFIED` in source here; commonly 4, `EXTERNAL`). + +## 5. Data model + +Format data model (`displayDescriptionData*.cs`): `TypeInfoDataBase` with sections for default settings (shape-selection directives, display-error and string-substitution settings, `autosize`), type groups, and view definitions (`name`, `appliesTo` type/group references, `mainControl` = table/list/wide/complex control body, `outOfBand`, `groupBy`). Formatting records (`FormatStartData` with shape header info, `GroupStartData`, `FormatEntryData` with payload, `GroupEndData`, `FormatEndData`) travel through the pipeline as `PSObject`s of internal types in the namespace `Microsoft.PowerShell.Commands.Internal.Format`. `OutCommandInner` keeps a `FormattedObjectsCache` and a `FormatMessagesContextManager` (callbacks: context creation, format start/end, group start/end, payload). + +## 6. Control flow + +```mermaid +flowchart TD + U[objects from pipeline] --> OD[Out-Default process] + OD --> OM[OutputManagerInner] + OM --> SP[SubPipelineManager: Out-LineOutput] + SP --> OC[OutCommandInner.ProcessRecord] + OC -->|raw object| FD[private Format-Default step] + FD --> VS[view selection: database, else properties] + VS --> REC[formatting records] + REC --> OC + OC -->|record| CACHE[FormattedObjectsCache group caching] + CACHE --> CTX[FormatMessagesContextManager] + CTX --> W[Table, List or Complex writer] + W --> LO[LineOutput: console, string or file] +``` + +**Table layout caching**: when the format-start record carries autosize information, the cache holds the first *n* objects of the group (`objectCount`) before writing, so column widths can be computed from them; when the header's first column width is 0 (auto), caching is enabled with a **300 ms** time window; otherwise records are written as they arrive. The sink may require buffering (`RequiresBuffering`), in which case playback of the cache is delegated to the sink (`ExecuteBufferPlayBack`); otherwise the cache is drained at end of input (src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394). + +## 7. Algorithms + +View selection and shape fallback are in §4. Column width computation (`ColumnWidthManager`), wrapping, truncation and the console paging logic are `UNVERIFIED` (VB-041). + +## 8. Observable behavior [SPEC] + +- Objects with no format data print as a table when they have at most the threshold number of display properties, else as a list; `Format-List`/`Format-Table`/`Format-Wide`/`Format-Custom` force a shape; `-View` picks a named view; `-Property` forces a property view. +- Table output of a first batch can be delayed up to about 300 ms to compute column widths when widths are automatic. +- Error records reaching `Out-Default` are formatted with the error view (`$ErrorView`, see errors document). +- `Out-Default -Transcript` writes objects to the transcript only. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`FormatErrorManager` collects property-expression failures and string-format errors and turns them into error records (`DrainFailedResultList`). Unknown view name → terminating error through `TerminatingErrorContext`. An out-of-order formatting record → format-state error. Objects that cannot be formatted fall back to `ToString` (`GenerateOutOfBandObjectAsToString`). Deserialized objects use the `Deserialized.` prefix fallback. + +## 10. Concurrency and threading + +Runs on the pipeline thread; `OutputManagerInner.StopProcessing` takes a lock and stops the sink (`_isStopped`), because stop requests arrive from another thread. The 300 ms grouping window is evaluated on arrival of records, not with a timer (`INFERENCE`; `FormattedObjectsCache` not read in full). + +## 11. Extension points and customization hooks + +Format data files (`Update-FormatData`, module `FormatsToProcess`, `types` + `format.ps1xml`), `PSStandardMembers` default display property set, `ToString` of objects, `$FormatEnumerationLimit`, `PSStyle` and `$PSStyle.OutputRendering`, custom `LineOutput` implementations (internal), `Out-*` cmdlets as sinks (`Out-File`, `Out-String`, `Out-Null`, `Out-Printer` where present). + +## 12. Dependencies + +Inbound: every interactive pipeline (host appends `Out-Default`), `Out-String`, `Out-File`, error display. Outbound: ETS property expressions (`PSPropertyExpression`), `LanguagePrimitives` string conversion, the host UI (`ConsoleLineOutput` uses host raw UI for width/height and paging), `PSStyle`, the format database (`TypeInfoDataBaseManager` held by the execution context: `FormatDBManager`). + +## 13. Performance notes + +Built-in format data is compiled to C# (`DefaultFormatters/*.cs`) instead of parsing XML at start-up; per-type view lookup walks the view list linearly on every record batch (`UNVERIFIED` caching); table width computation needs up to *n* objects or 300 ms of buffering, which delays first output. + +## 14. Security considerations + +Formatting runs user-influenced code: property expressions in views can contain script blocks, and `ToString` of arbitrary objects runs object code, so format data is a code-execution surface (format data from untrusted modules is loaded only when the module is imported; `UNVERIFIED` language-mode checks, Phase 5). Output sinks write to host and transcript, so redaction and `SecureString` handling matter here (`SecureString` prints as its type name). Escape sequences in object text can attack terminals; `PSStyle.OutputRendering` and string-decoration code (`StringDecorated`) exist for that (`UNVERIFIED` scope). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| remoting-deser | 10 | +| jea | 4 | +| logging | 3 | +| language-mode | 2 | +| app-control | 1 | +| securestring | 1 | +| **total** | 21 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/FormatAndOutput/common/Utilities/MshParameter.cs` | 9 | +| `src/System.Management.Automation/FormatAndOutput/DefaultFormatters/PowerShellCore_format_ps1xml.cs` | 4 | +| `src/System.Management.Automation/FormatAndOutput/out-console/OutConsole.cs` | 3 | +| `src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/XmlLoaderBase.cs` | 2 | +| `src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataXmlLoader.cs` | 1 | +| `src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs` | 1 | +| `src/System.Management.Automation/FormatAndOutput/common/Utilities/MshObjectUtil.cs` | 1 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: view selection and property evaluation run under the caller's language mode; display never executes code from format data that its policy forbids; terminal control sequences from data are neutralized before reaching a terminal unless rendering is explicitly enabled; transcripts receive the same redaction as the console. + +## 15. Enterprise considerations + +Security first: §14. Observability: format-view binding tracer (`format_view_binding`), formatting error collection. Supportability: `-View` and `Get-FormatData`/`Update-FormatData` allow inspection and override. Testability: formatting is deterministic given width; hard to unit-test because it depends on host width/height. Resource limits: group caching bounds memory by object count or time; no limit on line length except host width. Missing: structured (non-text) output channels for machine consumers (`ConvertTo-Json` is separate). + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Modules/Microsoft.PowerShell.Core/Out-Default.Tests.ps1` | 1 | 0 | 4 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Out-Host.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Format-Custom.Tests.ps1` | 3 | 2 | 11 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Format-List.Tests.ps1` | 3 | 0 | 23 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Format-Table.Tests.ps1` | 2 | 0 | 36 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Format-Wide.Tests.ps1` | 2 | 0 | 14 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Out-File.Tests.ps1` | 2 | 0 | 11 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Out-String.Tests.ps1` | 2 | 0 | 8 | +| `test/powershell/engine/Formatting/BugFix.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/Formatting/ErrorView.Tests.ps1` | 1 | 3 | 22 | +| `test/powershell/engine/Formatting/OutputRendering.Tests.ps1` | 1 | 0 | 4 | +| `test/powershell/engine/Formatting/PSStyle.Tests.ps1` | 3 | 0 | 37 | +| **total (12 files)** | 22 | 5 | 174 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `Core/Out-Default`, `Core/Out-Host`, `Utility/Out-String`, `Utility/Format-Custom`, `Utility/Format-Wide`, `Utility/Out-File`, `Utility/Format-Table`, `Utility/Format-List`, `engine/Formatting/`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: formatting is a separate pipeline stage that turns objects into *records* (data) and a second stage that turns records into text; this lets `Format-*` cmdlets and `Out-*` cmdlets be combined (formatting objects are what `Format-Table | Out-File` passes). +- `FACT-SRC`: the display database selects by type-name list, so inheritance order of type names (ETS) is the specificity rule. +- `INFERENCE`: the sub-pipeline and `CommandWrapper` exist to reuse cmdlet code paths without exposing internal commands in the command table. +- `FACT-SRC`: an empty per-type handler list in `SubPipelineManager` is an unused extension point. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTFMT-01** [REPLICATE] (area: NLD-RUN) Two-stage output: objects → formatting records (data) → line layout, with the record state machine; `Format-*` output can be piped to `Out-*`. Evidence: src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394. +- **LSN-RTFMT-02** [REPLICATE] (area: NLD-RUN) View selection by type-name list with perfect match then best match, then property-count fallback (table at or below the threshold, else list). Evidence: src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs::FormatViewManager.Initialize@L91-L204. +- **LSN-RTFMT-03** [ADAPT] (area: NLD-RUN) Replace the hidden sub-pipeline (`CommandWrapper` over `Out-LineOutput`) by a direct call to the formatter and sink objects, keeping the same observable output. Evidence: src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs::SubPipelineManager@L141-L326. +- **LSN-RTFMT-04** [ADAPT] (area: NLD-RUN) Keep table width auto-sizing but make the buffering window a configurable policy (count and time) instead of the fixed 300 ms. Evidence: src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394. +- **LSN-RTFMT-05** [REDESIGN] (area: NLD-RUN) Provide an optional structured output sink (typed records for machines) next to the text sink, so enterprise tooling does not parse text. DEV: DEV-FMT-160. Evidence: src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs::LineOutput@L215-L304. +- **LSN-RTFMT-06** [AVOID] (area: NLD-RUN) Do not carry the format XML loader family (`typeDataXmlLoader*`, schema validation) in the first version; ship built-in views as compiled data plus a small JSON/YAML view definition for extensions. DEV: DEV-FMT-161. Evidence: src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataXmlLoader.cs::TypeInfoDataBaseLoader@L40-L2234. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-040): `TypeMatch.PerfectMatch`/`BestMatch` ranking, `Complex` (custom) view semantics, `groupBy` handling, expression/script evaluation inside views, `PropertyCountForTable` default. +- `UNVERIFIED` (VB-041): `TableWriter`, `ListWriter`, `ComplexWriter`, `ColumnWidthManager`, `ConsoleLineOutput` (paging, wrapping), `FormattedObjectsCache` internals, `PSStyle`/`StringDecorated`, `OutputRendering` modes. +- `UNVERIFIED` (VB-042): the XML loader and schema validation (`typeDataXmlLoader*`, `XmlLoaderBase`), `FormatTable` (public API), built-in formatter data contents. + +Verification: `OutConsole.cs` lines 36–170, `BaseCommand.cs` (whole non-comment text), `OutputManager.cs` lines 19–326 (partly), `BaseOutputtingCommand.cs` lines 16–166, 256–400, `FormatViewManager.cs` lines 91–215, 399–467, `typeDataQuery.cs` lines 333–586 and `format-default.cs` were read; remaining files are named from the file list only. Ranges are in `T31_T39.csv`. diff --git a/docs-out/03-runtime/help-system.md b/docs-out/03-runtime/help-system.md new file mode 100644 index 0000000..cbadad9 --- /dev/null +++ b/docs-out/03-runtime/help-system.md @@ -0,0 +1,175 @@ +--- +id: RT-HELP-001 +title: Help system +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/help/HelpSystem.cs, src/System.Management.Automation/help/HelpProvider.cs, src/System.Management.Automation/help/CommandHelpProvider.cs, src/System.Management.Automation/help/HelpCommentsParser.cs, src/System.Management.Automation/help/HelpCommands.cs, src/System.Management.Automation/help/UpdatableHelpSystem.cs, src/System.Management.Automation/help/UpdateHelpCommand.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/help/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Help system + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. Help output is shaped by [formatting-and-output](formatting-and-output.md) (help views are format data); command lookup used by help is [command-discovery-and-invocation](command-discovery-and-invocation.md). + +## 1. Purpose and scope [SPEC] + +`Get-Help` finds documentation for a command, provider, alias, class, or conceptual topic and returns help objects. Sources: XML help files (MAML) shipped with modules, *comment-based help* inside functions and scripts, conceptual `about_*` text files, and auto-generated default help derived from command metadata. *Updatable help* downloads and installs help packages (`Update-Help`, `Save-Help`). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/help/HelpSystem.cs::HelpSystem@L88-L798 | class | provider list, request dispatch, exact and search modes, forwarding | B | +| src/System.Management.Automation/help/HelpProvider.cs::HelpProvider@L43-L254 | abstract class | provider contract | B | +| src/System.Management.Automation/help/HelpRequest.cs::HelpRequest@L22-L177 | class | target, category, flags, max results | B | +| src/System.Management.Automation/help/HelpInfo.cs::HelpInfo@L24-L268 | abstract class | a result, with forward target/category | B | +| src/System.Management.Automation/help/CommandHelpProvider.cs::CommandHelpProvider@L28-L1356 | class | cmdlet and function help (MAML + script help) | B | +| src/System.Management.Automation/help/HelpCommentsParser.cs::HelpCommentsParser@L23-L1225 | class | comment-based help parser | B | +| src/System.Management.Automation/help/MamlNode.cs::MamlNode@L67-L1251 | class | MAML XML reader | B | +| src/System.Management.Automation/help/DefaultCommandHelpObjectBuilder.cs::DefaultCommandHelpObjectBuilder@L43-L933 | class | synthesizes help from metadata | B | +| src/System.Management.Automation/help/HelpCommands.cs::GetHelpCommand@L27-L740 | cmdlet | `Get-Help` | B | +| src/System.Management.Automation/help/UpdatableHelpSystem.cs::UpdatableHelpSystem@L233-L1573 | class | HelpInfo.xml and package download/install | B | +| src/System.Management.Automation/help/UpdateHelpCommand.cs, SaveHelpCommand.cs | cmdlets | `Update-Help`, `Save-Help` | B | +| src/System.Management.Automation/help/AliasHelpProvider.cs, ProviderHelpProvider.cs, HelpFileHelpProvider.cs, PSClassHelpProvider.cs, DefaultHelpProvider.cs, ScriptCommandHelpProvider.cs, DscResourceHelpProvider.cs | classes | the other providers | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the provider chain and its order, search/forward semantics, help file lookup rules, comment-help keywords, updatable-help flow. Does not own: help content (shipped separately), rendering (format data), web access beyond the downloader. + +## 4. Key types and contracts [IMPL] + +- **Provider contract** (`HelpProvider`): `Name`, `HelpCategory` (flags), `ExactMatchHelp(request)`, `SearchHelp(request, searchOnlyContent)`, virtual `ProcessForwardedHelp(info, request)` and `Reset()`. +- **Provider chain** (`InitializeHelpProviders`), in order: alias, script-command (comment-based), command (MAML), provider, class (PS classes), help-file (about topics and file-based help), default. (DSC resource help is present in source but disabled in the list, with a comment citing a performance bug.) The chain is built once per `HelpSystem` (src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.InitializeHelpProviders@L563-L593). +- **Exact match** (`ExactMatchHelp`): for each provider whose category intersects the request's category, collect its results; every result is passed through *forwarding*; when a provider has produced at least one result and it is not the script-command provider, the search stops (so script help can be merged with the command provider's result, otherwise the first provider with a hit wins) (src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.ExactMatchHelp@L352-L379). +- **Forwarding**: a help object may name a forward target and category (for example proxy functions or comment keywords `.FORWARDHELPTARGETNAME`, `.FORWARDHELPCATEGORY`); `ForwardHelp` asks providers of the forward category to process it and recurses; if no provider handles the category the original result is returned; a result that forwards to its own request raises `CircularDependencyInHelpForwarding` (src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.ForwardHelp@L392-L435). +- **Command help lookup** (`CommandHelpProvider.GetHelpInfo`): for a cmdlet, check the in-memory cache by module and name; else find the cmdlet's help file (see below), load it into the cache, and read from cache or from the cmdlet metadata (auto-generated); for a function, load its declared help file if any; for any script-based command, parse comment-based help from its script block (also taking a `.LINK` URI as the online help URI). Candidates come from a `CommandSearcher` configured for exact match, filtered by session visibility for the request's origin; duplicate help names are skipped; `MaxResults` stops early (src/System.Management.Automation/help/CommandHelpProvider.cs::CommandHelpProvider.GetHelpInfo@L189-L410). +- **Help file search**: the file name is the cmdlet's declared help file (assembly-name based, `.dll-Help.xml`; a `.ni.dll` name is first tried as `.dll`). If the name is not an existing path, search directories are, in order: for a snap-in, the user help folder then the snap-in application base; for a module cmdlet, the user help folder for that module version then the module base; otherwise the user help folder, the shell's default path and the cmdlet assembly's folder. `MUIFileSearcher` locates the file for the current UI culture (falling back through parent cultures, `UNVERIFIED` exact order). Engine modules use a built-in map from module to help file under the culture folder (src/System.Management.Automation/help/CommandHelpProvider.cs::CommandHelpProvider.GetHelpFile@L494-L551). +- **Comment-based help keywords** (case-insensitive after a dot in a comment block): `SYNOPSIS`, `DESCRIPTION`, `PARAMETER `, `EXAMPLE`, `INPUTS`, `OUTPUTS`, `NOTES`, `LINK`, `COMPONENT`, `ROLE`, `FUNCTIONALITY`, `FORWARDHELPTARGETNAME`, `FORWARDHELPCATEGORY`, `REMOTEHELPRUNSPACE`, `EXTERNALHELP` (src/System.Management.Automation/help/HelpCommentsParser.cs::HelpCommentsParser.AnalyzeCommentBlock@L712-L808; the placement rules for the comment block — before the function, at the start of the body, or at script start — are `UNVERIFIED`, VB-043). +- **Updatable help**: `HelpInfo.xml` (validated against an embedded schema with `SupportedUICultures` → `UICultureName`, `UICultureVersion`) describes available help packages per culture; `Update-Help` resolves a module's `HelpInfoUri`, downloads the package for each requested UI culture (default: the current UI culture and its parents), checks versions, and installs; `Save-Help` stores packages for offline installation (src/System.Management.Automation/help/UpdatableHelpSystem.cs, src/System.Management.Automation/help/UpdateHelpCommand.cs; details `UNVERIFIED`, VB-044). + +## 5. Data model + +`HelpRequest`: target string, `HelpCategory` flags (alias, cmdlet, provider, general, FAQ, glossary, help file, class, DSC resource, default…), `CommandOrigin`, `MaxResults`, search options. `HelpInfo`: name, synopsis, category, components, role, functionality, full object (`PSObject` with `TypeNames` such as `HelpInfo`/`MamlCommandHelpInfo`), forward target/category, online URI. Providers keep caches (command cache keyed by help file identifier and command name; loaded-file set). `HelpSystem` keeps the last errors and last category, a search-path list built from loaded module bases, and a script-block token cache for comment help. + +## 6. Control flow + +```mermaid +flowchart TD + G[Get-Help target] --> R[HelpRequest] + R --> E[ExactMatchHelp over provider chain] + E -->|hit| F[ForwardHelp] + E -->|no hit and wildcard or not exact| S[SearchHelp over providers] + S --> F + F --> O[help objects to pipeline] + O --> FM[format views for help] +``` + +Ordered: validate request and help culture → exact match through providers in chain order → forward results → if nothing found, search mode (wildcards, content search) → default help when no target. `UNVERIFIED` in detail: the branch conditions between exact and search modes (`DoGetHelp`, lines not read). + +## 7. Algorithms + +Chain, forwarding and file search are in §4. Comment-based help parsing (tokens of the comment, section splitting by keyword line, parameter descriptions merged with those written next to parameters) is in `HelpCommentsParser` (`UNVERIFIED` in detail). + +## 8. Observable behavior [SPEC] + +- `Get-Help Name` returns the first provider's match; `-Full`, `-Examples`, `-Parameter`, `-Online`, `-ShowWindow` select views or actions; wildcards search. +- Functions with comment help return it without any file; `.EXTERNALHELP` points to an XML file; missing help yields auto-generated help (syntax only). +- `Update-Help` installs per-culture help for installed modules. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Provider errors are collected (`LastErrors`) and optionally shown in verbose mode (`VerboseHelpErrors`); a missing help file only traces; XML load errors are reported when `reportErrors` is set; circular forwarding raises an exception; invalid HelpInfo URI → `InvalidHelpInfoUri`; unsupported UI culture → `FailedToUpdateHelpWithLocaleNoUICulture`. + +## 10. Concurrency and threading + +Help runs on the pipeline thread; the downloader is asynchronous inside `UpdatableHelpSystem` (`UNVERIFIED` threading); provider caches are per-`HelpSystem` (per runspace) and not locked in the code read. + +## 11. Extension points and customization hooks + +Comment-based help and `.EXTERNALHELP`; `HelpInfoUri` and `HelpVersion` in module manifests; the user help folder (per-user installed help); `$env:`-independent culture via UI culture; custom help providers are internal (no public registration). + +## 12. Dependencies + +Inbound: `Get-Help`, `-?` handling (`CreateGetHelpCommandProcessor` in command processors), tab completion (help text), `Get-Command` online URIs. Outbound: `CommandSearcher`, session state visibility, module info (base paths, `HelpInfoUri`), script-block parsing, XML, web client for updates, file system. + +## 13. Performance notes + +Help files are loaded once into a cache per file; the token cache avoids re-parsing script blocks; default help is cheap; loading all help for a wildcard search is expensive. + +## 14. Security considerations + +Updatable help downloads files from a module-controlled URI and installs them under the module or user help folder: it is a network and file-write surface (`UNVERIFIED` validation of downloaded packages, signature or hash checks; Phase 5). Help text can reference online URIs that the shell opens only with `-Online`. Comment help is parsed, not executed. Help file lookup trusts the module folder. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| remoting-deser | 5 | +| jea | 3 | +| **total** | 8 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/help/ProviderContext.cs` | 4 | +| `src/System.Management.Automation/help/CommandHelpProvider.cs` | 1 | +| `src/System.Management.Automation/help/DscResourceHelpProvider.cs` | 1 | +| `src/System.Management.Automation/help/PSClassHelpProvider.cs` | 1 | +| `src/System.Management.Automation/help/ProviderHelpProvider.cs` | 1 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: downloaded help is integrity-checked (hash or signature) before installation; help lookup is read-only; online navigation requires explicit user action and uses a policy-approved opener. + +## 15. Enterprise considerations + +Security first: §14. Observability: help error tracer (`HelpErrorTracer`), `LastErrors`. Supportability: offline help via `Save-Help` and shares; per-culture versions. Testability: providers are small classes; engine tests exist. Resource limits: none on cache size. Missing: signed help packages, an air-gapped catalog format. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Scripting/ScriptHelp.Tests.ps1` | 3 | 28 | 92 | +| `test/powershell/engine/Help/HelpSystem.OnlineHelp.Tests.ps1` | 3 | 0 | 3 | +| `test/powershell/engine/Help/HelpSystem.Tests.ps1` | 21 | 2 | 30 | +| `test/powershell/engine/Help/UpdatableHelpSystem.Tests.ps1` | 11 | 0 | 2 | +| **total (4 files)** | 38 | 30 | 127 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `engine/Help/`, `Scripting/ScriptHelp`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: help is looked up by *providers in a fixed order* rather than a registry, so user-defined functions can shadow built-in help simply by being earlier in the command search. +- `FACT-SRC`: help for functions can be comment-based so documentation travels with the code. +- `INFERENCE`: DSC resource help was disabled for performance; the code remains. +- `FACT-SRC`: help is localized by UI culture with per-culture folders and versions. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTHELP-01** [REPLICATE] (area: NLD-TOOL) Comment-based help with the keyword set above and `.EXTERNALHELP`/forwarding, since scripts written for PowerShell carry it. Evidence: src/System.Management.Automation/help/HelpCommentsParser.cs::HelpCommentsParser.AnalyzeCommentBlock@L712-L808. +- **LSN-RTHELP-02** [ADAPT] (area: NLD-TOOL) Replace the fixed provider list with registered providers that declare category and priority, keeping the same default order. Evidence: src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.InitializeHelpProviders@L563-L593. +- **LSN-RTHELP-03** [ADAPT] (area: NLD-TOOL) Keep forwarding but make cycles a static check at load rather than a run-time exception. Evidence: src/System.Management.Automation/help/CommandHelpProvider.cs::CommandHelpProvider.ExactMatchHelp@L425-L472. +- **LSN-RTHELP-04** [REDESIGN] (area: NLD-SEC) Deliver updatable help as signed packages with a catalog and an offline mode, verified before install. DEV: DEV-HELP-165. Evidence: src/System.Management.Automation/help/UpdatableHelpSystem.cs::UpdatableHelpSystem@L233-L1573. +- **LSN-RTHELP-05** [AVOID] (area: NLD-TOOL) Do not port MAML as the authoring format or the DSC resource provider; use a lighter structured format (and the comment-based format) for built-in help. DEV: DEV-HELP-166. Evidence: src/System.Management.Automation/help/MamlNode.cs::MamlNode@L67-L1251. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-043): `DoGetHelp`/`SearchHelp` branch rules, comment-block placement, `MUIFileSearcher` culture fallback, `Get-Help` parameter handling, `ProviderHelpProvider`, `HelpFileHelpProvider` (about topics), `DefaultCommandHelpObjectBuilder`. +- `UNVERIFIED` (VB-044): `UpdatableHelpSystem` download/install/verification, `CabinetNativeApi`, `SaveHelpCommand`. + +Verification: `HelpSystem.cs` lines 158–260, 352–460, 563–640, `CommandHelpProvider.cs` lines 189–300, 425–640, `HelpCommentsParser.cs` keyword switch region (lines 726–790, 889–915), `UpdatableHelpSystem.cs`/`UpdateHelpCommand.cs` selected lines were read; other files are named from the file list. Ranges are in `T31_T39.csv`. diff --git a/docs-out/03-runtime/jobs-and-concurrency.md b/docs-out/03-runtime/jobs-and-concurrency.md new file mode 100644 index 0000000..f62c73c --- /dev/null +++ b/docs-out/03-runtime/jobs-and-concurrency.md @@ -0,0 +1,188 @@ +--- +id: RT-JOBS-001 +title: Jobs and concurrency +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/remoting/client/Job.cs, src/System.Management.Automation/engine/remoting/client/Job2.cs, src/System.Management.Automation/engine/remoting/client/JobManager.cs, src/System.Management.Automation/engine/remoting/client/JobSourceAdapter.cs, src/System.Management.Automation/engine/remoting/client/ThrottlingJob.cs, src/System.Management.Automation/engine/remoting/commands/JobRepository.cs, src/System.Management.Automation/engine/remoting/commands/StartJob.cs, src/System.Management.Automation/engine/hostifaces/PSTask.cs, src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs, src/System.Management.Automation/engine/InternalCommands.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/remoting/client/_folder.md, src-mirror/src/System.Management.Automation/engine/hostifaces/_folder.md] +generated_from: [data/types.json, data/enums.json, data/tests.json, data/enforcement-points.json] +--- + +# Jobs and concurrency + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. Threading inside one pipeline is described in [runspace-and-pipeline](runspace-and-pipeline.md) §10; runspace pools there too. Remoting transports are Phase 4. + +## 1. Purpose and scope [SPEC] + +Ways to run script concurrently: **jobs** (`Start-Job`: a background process; `Start-ThreadJob`: a module not in this repo's engine; remoting and workflow-style jobs through `Job2` sources), **`ForEach-Object -Parallel`** (script block per input object in pooled runspaces within the process), and **runspace pools** (SDK). This document covers the job object model and states, the job repository and source adapters, the process-based background job, the in-process parallel task pool, and the concurrency rules. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/remoting/client/Job.cs::Job@L408-L1719 | abstract class | job base: state, child jobs, six streams, events | B | +| src/System.Management.Automation/engine/remoting/client/Job.cs::JobState@L31-L90 | enum | job states | B | +| src/System.Management.Automation/engine/remoting/client/Job2.cs::Job2@L51-L447 | abstract class | adds async start/stop/suspend/resume APIs | B | +| src/System.Management.Automation/engine/remoting/client/JobManager.cs::JobManager@L24-L1054 | class | registry of job source adapters; `Get-Job`-style queries across sources | B | +| src/System.Management.Automation/engine/remoting/client/JobSourceAdapter.cs::JobSourceAdapter@L375-L544 | abstract class | a source of jobs by type name (a job provider) | B | +| src/System.Management.Automation/engine/remoting/commands/JobRepository.cs::JobRepository@L139-L188 | class | the session's table of jobs | B | +| src/System.Management.Automation/engine/remoting/commands/StartJob.cs::StartJobCommand@L22-L822 | cmdlet | `Start-Job` | B | +| src/System.Management.Automation/engine/remoting/client/Job.cs::PSRemotingJob@L1728-L2672 | class | job whose children run in (remote or child-process) runspaces | B | +| src/System.Management.Automation/engine/remoting/client/ThrottlingJob.cs::ThrottlingJob@L28-L1262 | class | parent job that throttles how many child jobs run at once | B | +| src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs::PowerShellProcessInstance@L16-L292 | class | the child `pwsh` process used by background jobs | B | +| src/System.Management.Automation/engine/hostifaces/PSTask.cs::PSTaskPool@L640-L987 | class | in-process task pool for `-Parallel` | B | +| src/System.Management.Automation/engine/hostifaces/PSTask.cs::PSTaskJob@L996-L1184 | class | job wrapper for `ForEach-Object -Parallel -AsJob` | B | +| src/System.Management.Automation/engine/InternalCommands.cs::ForEachObjectCommand@L65-L1271 | cmdlet | `ForEach-Object` including `-Parallel` | B | + +## 3. Responsibilities and non-responsibilities + +Owns: job state machine, job tables, throttling, child-process job runspaces, task pool and its data-stream merging. Does not own: the PSRP protocol (Phase 4), the `ThreadJob` module, scheduled jobs (module), runspace pools (T31). + +## 4. Key types and contracts [IMPL] + +- **`JobState`**: `NotStarted=0`, `Running`, `Completed`, `Failed`, `Stopped`, `Blocked`, `Suspended`, `Disconnected`, `Suspending`, `Stopping`, `AtBreakpoint` (src/System.Management.Automation/engine/remoting/client/Job.cs::JobState@L31-L90). Finished states are `Completed`, `Failed`, `Stopped`. +- **`Job`** has an auto-incremented `Id` (static seed), `InstanceId`, `Name` (auto-generated "Job" + number unless given), `Command`, `JobStateInfo`, `ChildJobs`, begin/end times, the streams (`Output`, `Error`, `Warning`, `Verbose`, `Debug`, `Progress`, `Information`) as `PSDataCollection`s, `HasMoreData`, `Location`, `StatusMessage`, abstract `StopJob()`, and events (`StateChanged`). `SetJobState(state, reason)` under a lock replaces the state info, stamps `PSBeginTime` on the first `Running` and `PSEndTime` on the first finished state, and for finished states closes all streams; it then raises `StateChanged` (handler exceptions are swallowed and traced) and, for finished states, sets the "finished" event (src/System.Management.Automation/engine/remoting/client/Job.cs::Job.SetJobState@L1386-L1462). A state in `Disconnected` or `Suspended` counts as "can be waited on" for waiters. +- **`Job2`** adds `StartJob`/`StartJobAsync`, `StopJobAsync`, `SuspendJob`, `ResumeJob`, `UnblockJob` and a `JobInvocationInfo`; `ContainerParentJob` aggregates child `Job2`s. `JobSourceAdapter`s register by type name in `JobManager` (public constructor required; constructed by reflection); `JobRepository` holds the session's jobs (lookup methods not read). +- **Background process job** (`Start-Job`): `PSRemotingJob` with child jobs backed by runspaces whose connection is a child `pwsh` process started with `-s -NoLogo -NoProfile` (server mode over standard streams); the parameters of `Start-Job` were not read (`UNVERIFIED`, VB-051); `PowerShellProcessInstance` starts the process and raises an event when it exits (src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs::PowerShellProcessInstance@L16-L292). +- **`ThrottlingJob`**: a parent that starts `StartableJob` children subject to a throttle limit and a queue; jobs are started as capacity frees (`UNVERIFIED` details). +- **`ForEach-Object -Parallel`**: parameters `Parallel` (script block), `ThrottleLimit` (default **5**), `TimeoutSeconds` (0 = none; a timer stops all tasks when it fires), `UseNewRunspace`, `AsJob`. `BeginProcessing` captures `$using:` values from the script block (`GetUsingValuesForEachParallel`), creates a `PSTaskDataStreamWriter` and a `PSTaskPool(ThrottleLimit, UseNewRunspace)`; `ProcessRecord` creates one `PSTask` per input object (with `$_` set, the current location path and the using-values map) and adds it to the pool, which **blocks the pipeline thread** when the pool is full (src/System.Management.Automation/engine/InternalCommands.cs::ForEachObjectCommand@L65-L1271, `PSTaskPool.Add@L762-L800`). `EndProcessing` waits for all tasks and drains the data-stream writer. +- **Task pool** (`PSTaskPool`): `size` limit; by default runspaces are *reused* from a queue (`_useRunspacePool`) unless `UseNewRunspace` creates a new one per task; a manual-reset event `_addAvailable` is reset when the pool is at its limit and set when a task completes; `_stopAll` unblocks adders. `PSTaskBase.Start(runspace)`: sets the task's location in the runspace, creates a `PowerShell` for it, wires stream-data-added handlers (`PSTask` forwards output/error/warning/verbose/debug/progress/information into the shared `PSTaskDataStreamWriter` as `PSStreamObject`s), adds the script text (`ToString()` of the block) with `$_`, binds the using-values map through the verbatim-argument parameter, and calls `BeginInvoke`. `SignalStop` calls `BeginStop`. Task ids come from a static counter; task runspaces are named "PSTask" (src/System.Management.Automation/engine/hostifaces/PSTask.cs::PSTaskBase@L321-L502). + +## 5. Data model + +Job: id, instance id, name, command, state info, child jobs, stream collections, begin/end times, `Location`, `StatusMessage`, finished wait handle, `PSJobTypeName`, and tracing. `JobRepository`: a list of jobs (keyed lookups). `JobManager`: dictionary of adapters by type name and a static map to reuse job ids across sessions by GUID. `PSTaskPool`: dictionary of active tasks by id, queue of idle runspaces, set of all active runspaces, counters. + +## 6. Control flow + +```mermaid +stateDiagram-v2 + [*] --> NotStarted + NotStarted --> Running: start + Running --> Completed: all work done + Running --> Failed: error + Running --> Stopping: StopJob + Stopping --> Stopped + Running --> Suspending: SuspendJob + Suspending --> Suspended + Suspended --> Running: ResumeJob + Running --> Blocked: waiting for input + Blocked --> Running + Running --> Disconnected: session lost + Disconnected --> Running: reconnect + Running --> AtBreakpoint: debugger stop + AtBreakpoint --> Running + Completed --> [*] + Failed --> [*] + Stopped --> [*] +``` + +`UNVERIFIED`: which transitions are enforced; the code read shows state setting but no transition table (`SetJobState` accepts any state). The diagram is the vocabulary of states (`INFERENCE` for edges). + +**Parallel run, ordered**: `ForEach-Object -Parallel` — begin: build pool and writer; each input object: build task, wait for a free slot, start task in a runspace; meanwhile task output flows to the writer which the cmdlet drains on its own thread (so output order is completion order, not input order); end: wait for tasks, flush streams, dispose pool. + +## 7. Algorithms + +Throttle: block adder when `taskPool.Count == sizeLimit`; unblock on completion. Timeout: a timer with due time `TimeoutSeconds × 1000` stops all tasks. Job id reuse: `JobManager` keeps ids by GUID so a re-attached job keeps its id (`UNVERIFIED`). + +## 8. Observable behavior [SPEC] + +- `EXTERNAL` (public cmdlet documentation, cmdlets not read): `Start-Job` returns a job object immediately; `Receive-Job` collects output; `Wait-Job` waits for finished states; `Stop-Job` stops; `Remove-Job` deletes. +- `FACT-SRC`: background jobs run in a separate `pwsh` process started with `-NoProfile`; `INFERENCE`: they therefore do not inherit the parent's variables. +- `FACT-SRC`: `ForEach-Object -Parallel` runs at most 5 script blocks at a time by default, in separate runspaces, with `$using:` for outer values; output order follows completion. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Stream data from tasks is marshalled to the cmdlet thread; a task failure is reported as an error record; timeouts stop tasks and end the pipeline with an error (`UNVERIFIED` text); a job in `Disconnected` can be reconnected; job disposal while running requires stop. Non-finished state changes are not validated. + +## 10. Concurrency and threading + +Locks: `Job.syncObject` guards state; `PSTaskPool._syncObject` guards the task table; concurrent collections for idle/active runspaces; manual-reset events for slot availability and stop; static id counters use `Interlocked`. Events (`StateChanged`) run on whichever thread changes the state. Task data events fire on the task's thread and are queued into a thread-safe writer drained by the cmdlet thread, which keeps all pipeline writes on the pipeline thread (see write-permission rule in [cmdlet-infrastructure](cmdlet-infrastructure.md)). + +## 11. Extension points and customization hooks + +Custom job types (derive from `Job`/`Job2`), `JobSourceAdapter` registration (internal in this version), `Register-ObjectEvent -AsJob` style event jobs (`PSEventJob`, see [eventing](eventing.md)), `-InitializationScript`, `-ThrottleLimit`, `-UseNewRunspace`, `-TimeoutSeconds`. + +## 12. Dependencies + +Inbound: job cmdlets, `ForEach-Object`, event subscriptions with actions, debugger (`DebugJob`). Outbound: runspaces and `PowerShell` (T31), remoting client (child-process transport, Phase 4), `PSDataCollection`, session state (`JobRepository`/`JobManager` on the runspace). + +## 13. Performance notes + +Process jobs pay process start-up (hundreds of ms); `-Parallel` reuses runspaces by default to avoid creating one per object, with `UseNewRunspace` trading speed for isolation; each task re-parses the script text (`ToString()` of the block is added as script). + +## 14. Security considerations + +A background job is a new process running as the same user with a fresh session: it does **not** inherit the language mode or restricted session state of the parent unless the engine passes it (`UNVERIFIED`: lockdown propagation, Phase 5). `ForEach-Object -Parallel` runspaces are created from the current session's initial state (`UNVERIFIED` exact policy inheritance). `$using:` values are copied by reference/serialization, which can expose objects across runspaces. Job output streams are retained in memory until received or removed. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 10 | +| app-control | 2 | +| **total** | 12 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/hostifaces/PSTask.cs` | 6 | +| `src/System.Management.Automation/engine/remoting/commands/StartJob.cs` | 6 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: child runspaces and processes inherit the parent's language mode, visibility and policy by default and can only be more restricted; `$using` captures follow the same trust rules as module exports; jobs have quotas (count, memory) and an owner recorded in audit logs. + +## 15. Enterprise considerations + +Security first: §14. Observability: job states, begin/end times, streams per job, tracing in job code. Supportability: `Get-Job` over multiple sources; job sources are pluggable. Testability: job logic is exercised via cmdlets. Resource limits: throttle limit per `-Parallel` (default 5) and `ThrottlingJob`; no global cap on jobs or memory. Missing: quotas, job priorities, durable jobs across restarts (scheduled jobs are a module), structured audit records. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Scripting/ForeachParallel.Tests.ps1` | 1 | 5 | 10 | +| `test/powershell/Language/Scripting/SwitchParallel.Tests.ps1` | 1 | 3 | 5 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/ForEach-Object.Tests.ps1` | 1 | 0 | 5 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Job.Tests.ps1` | 4 | 7 | 30 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Foreach-Object-Parallel.Tests.ps1` | 5 | 0 | 55 | +| `test/powershell/Modules/ThreadJob/ThreadJob.Tests.ps1` | 2 | 0 | 27 | +| `test/powershell/engine/Job/Jobs.Tests.ps1` | 1 | 7 | 32 | +| **total (7 files)** | 15 | 22 | 164 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `Core/Job.Tests`, `Core/ForEach-Object.Tests`, `Modules/ThreadJob/`, `Foreach-Object-Parallel`, `engine/Job/`, `ForeachParallel`, `SwitchParallel`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: process isolation for background jobs (safe against crashes and state leaks, slow, no object sharing). +- `FACT-SRC`: in-process parallelism through runspace pools (fast, isolated sessions, but objects are shared through `$using`). +- `INFERENCE`: blocking the pipeline thread when the pool is full gives natural back-pressure for `-Parallel`. +- `FACT-SRC`: jobs unify different execution backends behind one object with child jobs and streams. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTJOB-01** [REPLICATE] (area: NLD-RUN) One `Job` abstraction with child jobs, per-stream collections, finished-state semantics and a state-changed event. Evidence: src/System.Management.Automation/engine/remoting/client/Job.cs::Job.SetJobState@L1386-L1462. +- **LSN-RTJOB-02** [REPLICATE] (area: NLD-RUN) `ForEach-Object -Parallel` semantics: throttle default 5, `$using:` capture, completion-order output, timeout. Evidence: src/System.Management.Automation/engine/InternalCommands.cs::ForEachObjectCommand@L65-L1271. +- **LSN-RTJOB-03** [ADAPT] (area: NLD-RUN) Add an explicit transition table to the job state machine (the source accepts any transition). Evidence: src/System.Management.Automation/engine/remoting/client/Job.cs::Job.SetJobState@L1386-L1462. +- **LSN-RTJOB-04** [REDESIGN] (area: NLD-SEC) Child processes and parallel runspaces inherit policy, language mode and visibility from the parent as a hard rule, plus per-job quotas. DEV: DEV-JOB-190. Evidence: src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs::PowerShellProcessInstance@L16-L292. +- **LSN-RTJOB-05** [AVOID] (area: NLD-RUN) Do not port `ThrottlingJob`, `ContainerParentJob` and the job source-adapter registry in the first version; one local job type plus the parallel pool suffices. DEV: DEV-JOB-191. Evidence: src/System.Management.Automation/engine/remoting/client/ThrottlingJob.cs::ThrottlingJob@L28-L1262. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-051): `PSRemotingJob`/`PSRemotingChildJob` internals, `ThrottlingJob`, `Job2` async methods, `JobManager` query methods, `StartJob` parameter handling and lockdown inheritance, `PSTaskJob`/`PSTaskChildJob`, `ForEachObjectCommand` lines outside the parallel setup. +- `UNVERIFIED` (VB-052): the `ThreadJob` module (outside this engine), scheduled jobs, workflow jobs. + +Verification: `Job.cs` lines 408–470, 1386–1462 and the `JobState` enum, `JobManager.cs` lines 24–80, `PSTask.cs` lines 23–140, 321–500, 640–800, `InternalCommands.cs` parallel-related lines (240–271, 377–470, 543–565 via grep), `PowerShellProcessInstance.cs` argument lines were read; the rest is from signatures. Ranges in `T31_T39.csv`. diff --git a/docs-out/03-runtime/modules.md b/docs-out/03-runtime/modules.md new file mode 100644 index 0000000..5830786 --- /dev/null +++ b/docs-out/03-runtime/modules.md @@ -0,0 +1,198 @@ +--- +id: RT-MODULES-001 +title: Modules +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-MOD-001] +sources: [src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs, src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs, src/System.Management.Automation/engine/Modules/PSModuleInfo.cs, src/System.Management.Automation/engine/Modules/AnalysisCache.cs, src/System.Management.Automation/engine/Modules/ModuleUtils.cs, src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs, src/System.Management.Automation/engine/Modules/ExportModuleMemberCommand.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/Modules/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Modules + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `CORE` for script modules and manifests, binary modules are `EXTENDED`. Where modules enter command lookup: [command-discovery-and-invocation](command-discovery-and-invocation.md) (auto-loading, auto-discovery). Module scope and session state: [execution-context-and-session-state](execution-context-and-session-state.md). Script blocks bound to modules: [SCRIPT-BLOCKS](../02-language/SCRIPT-BLOCKS.md). + +## 1. Purpose and scope [SPEC] + +A *module* packages commands, variables, types, formats and scripts under a name and version, with its own session state so internals stay private. Kinds: *script module* (`.psm1`), *binary module* (`.dll` containing cmdlets/providers), *manifest module* (`.psd1` describing and wiring the others), *dynamic module* (`New-Module`, created from a script block), *CIM-cmdlet module* (`.cdxml`). This document covers the manifest model, module path resolution, the load and import algorithm, export and import rules, auto-loading and the analysis cache, and removal. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase@L40-L7478 | abstract class | base of `Import-Module`, `Get-Module`, `New-Module`, `Remove-Module`...: loading, manifests, imports, removal | B | +| src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs::ImportModuleCommand@L44-L2183 | cmdlet | `Import-Module` and its parameter sets (name, path, assembly, PSSession, CIM, WinCompat) | B | +| src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo@L23-L1554 | class | the module object: name, version, path, exports, session state, nested/required modules | B | +| src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics@L29-L1725 | class | `$ExecutionContext.Modules`: module table, path computation, export engine, version matching | B | +| src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCache@L31-L643 | static class | exported-command analysis cache for auto-discovery | B | +| src/System.Management.Automation/engine/Modules/ModuleUtils.cs::ModuleUtils@L14-L648 | static class | module file discovery and version folders | B | +| src/System.Management.Automation/engine/Modules/ModuleSpecification.cs::ModuleSpecification@L32-L339 | class | name/GUID plus version constraints | B | +| src/System.Management.Automation/engine/Modules/ExportModuleMemberCommand.cs::ExportModuleMemberCommand@L22-L190 | cmdlet | `Export-ModuleMember` | B | +| src/System.Management.Automation/engine/Modules/NewModuleManifestCommand.cs, TestModuleManifestCommand.cs, GetModuleCommand.cs, RemoveModuleCommand.cs, NewModuleCommand.cs | cmdlets | manifest creation/validation, listing, removal, dynamic modules | B | +| src/System.Management.Automation/engine/Modules/ScriptAnalysis.cs::ScriptAnalysis@L17-L116 | class | static analysis of script module exports | B | + +## 3. Responsibilities and non-responsibilities + +Owns: module file formats and discovery order, manifest keys and validation, version selection, the load algorithm, export lists, import (prefix, visibility, clobber), removal, auto-load/analysis cache. Does not own: the cmdlets' command-line surface (Phase 4 families), session-state tables themselves (T29), help lookup beyond `HelpInfoUri` (help document), remoting import (PSSession import, Phase 4). + +## 4. Key types and contracts [IMPL] + +- **Module types** (`ModuleType`): `Script=0`, `Binary=1`, `Manifest=2`, `Cim=3` (src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::ModuleType@L1559-L1578). +- **Manifest keys** (the list the engine knows): `ModuleToProcess`, `RootModule`, `NestedModules`, `GUID`, `Author`, `CompanyName`, `Copyright`, `ModuleVersion`, `Description`, `PowerShellVersion`, `PowerShellHostName`, `PowerShellHostVersion`, `CLRVersion`, `DotNetFrameworkVersion`, `ProcessorArchitecture`, `RequiredModules`, `TypesToProcess`, `FormatsToProcess`, `ScriptsToProcess`, `PrivateData`, `RequiredAssemblies`, `ModuleList`, `FileList`, `FunctionsToExport`, `VariablesToExport`, `AliasesToExport`, `CmdletsToExport`, `DscResourcesToExport`, `CompatiblePSEditions`, `HelpInfoURI`, `DefaultCommandPrefix` (src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase@L40-L7478, the `ModuleManifestMembers` table near line 251). A required-module entry is a string or a hashtable with `ModuleName`, `GUID`, `ModuleVersion` (plus required/maximum version). +- **Flags**: `ManifestProcessingFlags` = `WriteErrors=1`, `NullOnFirstError=2`, `LoadElements=4`, `WriteWarnings=8`, `Force=0x10`, …; `ImportModuleOptions` (no-clobber, local, service-side import, …). +- **Implicit module file extensions** tried in order when a name has no extension: `.psd1`, `.psm1`, `.cdxml`, `.ni.dll`, `.dll`, `.exe` (`PSModuleExtensions`; the `.ni.dll` precedes `.dll` so the native image wins) (src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.PSModuleExtensions@L896-L905). +- **`LoadUsingExtensions(name, baseName, extension, …)`**: for each candidate extension, resolve the file path (skip if unresolved or if it is the module currently being processed); if the module is already in the module table and loading is not forced and the loaded module satisfies the constraints, **re-import its members** (prefix, options), write it out if `-PassThru`/`-AsCustomObject`, and return it; else if the file exists, remove an already-loaded satisfying module when `-Force`, then call `LoadModule` (src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingExtensions@L5422-L5518). +- **Multi-version folders**: `LoadUsingMultiVersionModuleBase(moduleBase)` iterates version sub-folders of the module base (`ModuleUtils.GetModuleVersionSubfolders`, order from that function — `UNVERIFIED`, expected highest first), skips versions that do not satisfy the required/minimum/maximum version, builds `//.psd1`, accepts the folder only if the manifest's `ModuleVersion` equals the folder version (else it is skipped with a verbose message), and loads the first valid one (src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingMultiVersionModuleBase@L433-L478). +- **Module path** (`$env:PSModulePath`): `GetModulePath(includeSystem, context)` splits the variable (or computes the default when unset), resolves each entry (UNC paths kept as is; file-system provider paths resolved and de-duplicated case-insensitively; non-existing directories dropped when the provider is not loaded), and appends the PowerShell home `Modules` folder when `includeSystem` is true. The default value is built from: the per-user module folder (`~/.local/share/powershell/Modules` style on Unix via XDG, `Documents\PowerShell\Modules` on Windows), the shared folder (`/usr/local/share/powershell/Modules` on Unix, `Program Files\PowerShell\Modules` on Windows) and the PowerShell home `Modules` folder; on Windows the Windows PowerShell system path is also added for compatibility (src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.GetModulePath@L1397-L1419, GetPersonalModulePath@L965-L975, GetSharedModulePath@L1017-L1031; exact default composition in `SetModulePath`/`GetModulePath(string, …)` `UNVERIFIED`, VB-045). +- **Export** (`ExportModuleMembers`): sets the session state's "use export list" flag; for each category with patterns: *functions* in the module scope's function table (skipping `AllScope` items) that match any wildcard are added to `ExportedFunctions` (sorted by name, duplicates removed); *cmdlets* (compiled exports and module-scope cmdlets, minus a do-not-export list); *variables* (skipping `AllScope` and built-in variable names); *aliases* (compiled alias exports and module-scope aliases, skipping `AllScope`) (src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.ExportModuleMembers@L1529-L1689). A module that never exports and has no manifest export lists exports all functions, cmdlets and aliases by default (`UNVERIFIED` default for variables: none; see `PSModuleInfo` and `ModuleCmdletBase.LoadModule`). +- **Import** (`ImportModuleMembers(sourceModule, prefix, options)`): imports cmdlets, aliases, functions and variables the module exports into the target session state. Order: cmdlets, aliases, functions, variables. With `-Prefix` (or the manifest's `DefaultCommandPrefix`) names get the prefix after the verb (`Get-Foo` → `Get-PrefixFoo`) and aliases pointing to prefixed commands are rewritten; with `-NoClobber` commands/variables whose names already exist in the target are skipped (verbose message); items go to the module scope of the target (or the current scope with `-Local`); `-Global`/`-Scope` select the target session state; if the importing cmdlet is private or the target defaults to private visibility, imported commands become private. Unless `-DisableNameChecking`, the first non-approved verb or invalid noun triggers one warning, the rest verbose messages (`Sort-Object` and `Tee-Object` are exempt) (src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.ImportModuleMembers@L7022-L7251, `ValidateCommandName@L7375-L7458`). +- **Analysis cache** (`AnalysisCache.GetExportedCommands(path, testOnly, context)`): returns the exported-command map (`name → CommandTypes`) for a module file from a cache keyed by path and validated by the file's last write time; on a miss, analysis depends on the extension: `.psd1` — read a fast subset of manifest properties and the export lists (with a check that a versioned folder matches the manifest version), `.psm1` — static script analysis, `.cdxml`, `.dll`, `.exe` — binary/cdxml analysis; a result queues serialization of the cache to disk after a delay of 10 seconds; no analysis when `testOnly` (src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCache.GetExportedCommands@L44-L96, `AnalysisCacheData@L645-L1131`). + +**Module load, export and auto-load rules (CF-MOD-001).** (1) A module name without a path is searched in each `PSModulePath` entry in order (entries marked to skip the system folder are skipped; on Unix the folder name is matched case-insensitively): in an entry, the module base is `/`; its version sub-folders are tried first (the first whose version satisfies the constraints and equals its manifest's `ModuleVersion` is loaded); if none loads, the files `//` + extension are tried in the order `.psd1`, `.psm1`, `.cdxml`, `.ni.dll`, `.dll`, `.exe` (a name that already carries a module extension is tried as given); the first successful load ends the search and its exports are added to the analysis cache. (2) A manifest is read as data and its keys are checked against the known key list (§4); the processing of the root module, nested modules, required modules (version-constrained; a cyclic-requirement check exists), required assemblies, scripts to process, types and formats follows, in an order that is `UNVERIFIED` (VB-046); how the manifest export lists combine with a script module's own exports is `UNVERIFIED` (VB-046). (3) A module already loaded that satisfies the requested constraints is not loaded again unless `-Force`; only its members are re-imported with the requested prefix and options. (4) Each module has its own session state; exported commands are copied into the importer's session state. (5) `Export-ModuleMember` selects exports by wildcard patterns per category (functions, cmdlets, variables, aliases), skipping `AllScope` items and built-in variable names. (6) Import applies `-Prefix`/`DefaultCommandPrefix`, `-NoClobber`, visibility and the verb/noun checks, in that order: cmdlets, aliases, functions, variables. (7) **Auto-loading** happens during command lookup: a module-qualified name loads exactly that module; any other unknown command name is looked up in the analysis cache of every module file on `PSModulePath` and the first module exporting a command of that exact name is imported (`$PSModuleAutoLoadingPreference`: `None`, `ModuleQualified`, `All`). (8) Removal of a module (`RemoveModule`) is `UNVERIFIED` here (VB-046). + +## 5. Data model + +`PSModuleInfo`: name, path, base, version, GUID, description, author, module type, root module, nested modules, required modules, exported functions/cmdlets/aliases/variables/commands (dictionaries), `SessionState` (the module's), `ModuleBase`, `PrivateData`, `ExperimentalFeatures`, `CompatiblePSEditions`, `AccessMode`, `HelpInfoUri`, and fields for compiled exports. `ModuleIntrinsics` holds the module table (`ModuleTable`: path → module), the nesting depth counter (to limit recursion), and the "previously imported" map used to prefer a module when a command exists in several. Session state fields: `UseExportList`, `FunctionsExported`, `FunctionsExportedWithWildcard`, `ExportedFunctions/Aliases/Variables`, `Module`, `ModuleScope`. + +## 6. Control flow + +```mermaid +flowchart TD + IM[Import-Module name] --> P[resolve candidate paths over PSModulePath] + P --> V[version folders and constraints] + V --> LX[LoadUsingExtensions: psd1, psm1, cdxml, ni.dll, dll, exe] + LX -->|already loaded and satisfying| RI[re-import members] + LX -->|file exists| LM[LoadModule] + LM --> MF{extension} + MF -->|psd1| MAN[load manifest data, validate, process nested, required, scripts, types, formats] + MF -->|psm1| SM[create module session state, run script, collect exports] + MF -->|dll| BM[load assembly, register cmdlets and providers] + MAN --> EX[apply export lists] + SM --> EX + BM --> EX + EX --> IMP[ImportModuleMembers: prefix, clobber, visibility] + RI --> IMP +``` + +## 7. Algorithms + +Load, export, import and analysis are described in §4 and CF-MOD-001. Detailed manifest processing (`LoadModuleManifest`, ~2,000 lines) is `UNVERIFIED` (VB-046): key validation order, handling of `NestedModules` vs `RootModule`, `RequiredAssemblies`, `ScriptsToProcess`, `CompatiblePSEditions` and `PowerShellVersion` checks, and the required-module cycle check (`HasRequiredModulesCyclicReference`). + +## 8. Observable behavior [SPEC] + +- `Import-Module Name` finds the highest-version valid module in `PSModulePath` order; a module-qualified call `Name\Command` imports on demand; a plain unknown command is searched in all installed modules' exports. +- Functions not exported stay private to the module; module-scoped variables are invisible outside; `Export-ModuleMember` limits exports; manifest lists win. +- `Get-Module` lists loaded modules; `-ListAvailable` lists installed ones from the path; `Remove-Module` unloads. +- A warning appears the first time a module exports a command with an unapproved verb. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Module not found → `Modules_ModuleNotFound`-style errors; version-folder mismatch → skipped with verbose; invalid manifest member → `InvalidModuleManifest...` errors via `WriteInvalidManifestMemberError`; cyclic `RequiredModules` rejected; name conflict between parameter set or alias and parameter (not here); `-AsCustomObject` with a binary module → `Modules_CantUseAsCustomObjectWithBinaryModule`; dot-sourcing disallowed in some contexts (`CheckForDisallowedDotSourcing`); nesting depth limit (`IncrementModuleNestingDepth`). + +## 10. Concurrency and threading + +The module table lives on the execution context (single pipeline thread). The analysis cache is static and shared: concurrent dictionaries, background serialization task with a delay, and a concurrent set of modules being analyzed to avoid re-entrancy (`s_modulesBeingAnalyzed`). Auto-discovery runs on the pipeline thread. + +## 11. Extension points and customization hooks + +Manifests (`.psd1`) with `PrivateData`; `$PSModulePath`; `$PSModuleAutoLoadingPreference`; `InitialSessionState` modules and `ImportPSModule`; module logging policy (`SetModuleLoggingInformation`: policy-controlled lists of module names); `DefaultCommandPrefix`; script blocks `OnRemove` handlers (`PSModuleInfo.OnRemove`; `UNVERIFIED`); `ScriptsToProcess`; `FormatsToProcess`/`TypesToProcess`. + +## 12. Dependencies + +Inbound: command discovery (auto-load/discovery), `Import-Module`/`Get-Module`/`Remove-Module`, `#requires -Modules`, `InitialSessionState`. Outbound: session state (module scopes, tables), the parser (manifest data and script analysis), execution policy / authorization (script modules), type/format updaters, assembly loading, the file system, help (`HelpInfoUri`), experimental features. + +## 13. Performance notes + +Module discovery reads the file system on every miss; auto-discovery uses the analysis cache to avoid loading modules, and persists it to disk lazily. Binary module analysis may require reflection over assemblies. `PSModuleExtensions` order puts `.psd1` first to prefer manifests (cheap data load). Multi-version folders cost one manifest read per version considered. + +## 14. Security considerations + +Modules are code loaded from the file system and executed with the caller's rights, so the module path is a trust boundary: whoever can write a folder on `PSModulePath` (or earlier in it) can shadow commands (auto-discovery imports the *first* module that exports the name). Enforced here: execution policy and signature checks when loading scripts (`GetScriptInfoForFile(..., checkExecutionPolicy)`); manifests are parsed as restricted data; command visibility for imported commands in restricted sessions; module logging by policy; application-control language-mode handling for modules (modules defined in a trusted state keep it; `UNVERIFIED` where, Phase 5); `RequiredAssemblies` loads assemblies (policy-gated, `UNVERIFIED`). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 36 | +| app-control | 31 | +| logging | 17 | +| execution-policy | 2 | +| trust-boundary | 1 | +| **total** | 87 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs` | 65 | +| `src/System.Management.Automation/engine/Modules/ExportModuleMemberCommand.cs` | 7 | +| `src/System.Management.Automation/engine/Modules/NewModuleCommand.cs` | 6 | +| `src/System.Management.Automation/engine/Modules/PSModuleInfo.cs` | 5 | +| `src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs` | 3 | +| `src/System.Management.Automation/engine/Modules/ImportModuleCommand.cs` | 1 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: module identity is (name, version, GUID, hash) with a signed or allow-listed catalog; the search path is configured per policy and ordered by trust; auto-load is off by default for untrusted paths; imports record who loaded what from where in the audit log; a module cannot weaken the importing session's language mode or visibility. + +## 15. Enterprise considerations + +Security first: §14. Observability: module logging policy, verbose messages for each import/skip, ETW events around analysis. Supportability: manifests carry version, GUID, edition compatibility and help URI; `Test-ModuleManifest` validates. Testability: module loading is observable through cmdlets; internal functions are large and entangled (the 7,500-line base class). Resource limits: nesting depth limit; unbounded analysis cache. Missing: lock files / reproducible resolution, per-module resource limits, package source integration (separate PowerShellGet). + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Modules/Microsoft.PowerShell.Core/Get-Module.Tests.ps1` | 2 | 2 | 18 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Import-Module.Tests.ps1` | 8 | 2 | 19 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/ModuleManifest.Tests.ps1` | 5 | 0 | 5 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Remove-Module.Tests.ps1` | 6 | 0 | 16 | +| `test/powershell/engine/Module/IsolatedModule.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/engine/Module/ModulePath.Tests.ps1` | 3 | 0 | 12 | +| `test/powershell/engine/Module/ModuleSpecification.Tests.ps1` | 1 | 3 | 11 | +| `test/powershell/engine/Module/NewModuleManifest.Tests.ps1` | 2 | 0 | 6 | +| `test/powershell/engine/Module/SubmodulePathInManifest.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/engine/Module/TestModuleManifest.Tests.ps1` | 3 | 0 | 15 | +| `test/powershell/engine/Module/UpdateModuleManifest.Tests.ps1` | 1 | 0 | 1 | +| **total (11 files)** | 33 | 7 | 107 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `Core/ModuleManifest`, `Core/Get-Module`, `Core/Remove-Module`, `Core/Import-Module`, `engine/Module/`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: loading is "find file by extension order" rather than a registry, which makes folder layout the interface. +- `FACT-SRC`: the manifest is data (restricted language), the script module is code; the manifest's export lists are mirrored in the analysis cache so discovery does not execute code. +- `INFERENCE`: re-importing members of an already-loaded module instead of reloading gives cheap repeated `Import-Module` and supports different prefixes in different scopes. +- `FACT-SRC`: version sub-folders must match the manifest version, protecting against copied folders. +- `FACT-SRC`: a hidden global cache persisted with a 10-second delay trades freshness for speed (validated by last write time). + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTMOD-01** [REPLICATE] (area: NLD-MOD) Module = folder + manifest + optional root script/binary, with per-module session state and an export list; manifest is data, not code. Evidence: src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingExtensions@L5422-L5518. +- **LSN-RTMOD-02** [REPLICATE] (area: NLD-MOD) Version-folder rule (folder version equals manifest version) and constraint matching with required/minimum/maximum. Evidence: src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingMultiVersionModuleBase@L433-L478. +- **LSN-RTMOD-03** [REPLICATE] (area: NLD-MOD) Export engine with wildcard patterns per category and manifest lists overriding script exports. Evidence: src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.ExportModuleMembers@L1529-L1689. +- **LSN-RTMOD-04** [ADAPT] (area: NLD-MOD) Keep the exported-command analysis cache but make it a versioned, signed, explicitly refreshed index (see DEV-DISC-120), validated by content hash rather than last write time. Evidence: src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCache.GetExportedCommands@L44-L96. +- **LSN-RTMOD-05** [REDESIGN] (area: NLD-SEC) Module search path entries carry trust levels, and auto-load applies only from trusted entries unless policy allows more. DEV: DEV-MOD-170. Evidence: src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.GetModulePath@L1397-L1419. +- **LSN-RTMOD-06** [ADAPT] (area: NLD-MOD) Split the 7,500-line loader class by responsibility: path resolution, manifest reader, module factory per kind, export engine, importer, remover. Evidence: src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase@L40-L7478. +- **LSN-RTMOD-07** [AVOID] (area: NLD-MOD) Do not implement Windows PowerShell compatibility imports (`ImportModulesUsingWinCompat`), `.cdxml` CIM modules or DSC resource exports in the first version. DEV: DEV-MOD-171. Evidence: src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.ImportModulesUsingWinCompat@L4972-L5006. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-045): `SetModulePath` and the default `PSModulePath` composition, `ModuleUtils.GetModuleVersionSubfolders` ordering, `ModuleSpecification` matching rules (`IsVersionMatchingConstraints`). +- `UNVERIFIED` (VB-046): `LoadModuleManifest` (lines 1528–3580), `LoadModule` (5627–6120), `RemoveModule` (5022–5270), required-module resolution (3822–4210), binary module loading, `PSModuleInfo` members, `Import-Module` parameter handling. +- `UNVERIFIED` (VB-047): where language-mode and application-control rules are applied to module loading. + +Verification: `ModuleCmdletBase.cs` lines 251–285 (manifest keys), 433–478, 5422–5520, 7022–7252 (selected), 7375–7458, `ModuleIntrinsics.cs` lines 896–905, 965–1031, 1397–1530, 1529–1695, `AnalysisCache.cs` lines 44–140 (selected) and 645–690 (selected) were read. Everything else is from signatures. Ranges are in `T31_T39.csv`. diff --git a/docs-out/03-runtime/parameter-binding.md b/docs-out/03-runtime/parameter-binding.md new file mode 100644 index 0000000..e618411 --- /dev/null +++ b/docs-out/03-runtime/parameter-binding.md @@ -0,0 +1,191 @@ +--- +id: RT-BIND-001 +title: Parameter binding +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-BND-001] +sources: [src/System.Management.Automation/engine/CmdletParameterBinderController.cs, src/System.Management.Automation/engine/ParameterBinderController.cs, src/System.Management.Automation/engine/ParameterBinderBase.cs, src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs, src/System.Management.Automation/engine/CompiledCommandParameter.cs, src/System.Management.Automation/engine/ParameterSetSpecificMetadata.cs, src/System.Management.Automation/engine/CommandProcessor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/members-tierA.json, data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Parameter binding + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Where binding sits in the command lifecycle: [runspace-and-pipeline](runspace-and-pipeline.md) (CF-LCY-001). Cmdlet base classes: [cmdlet-infrastructure](cmdlet-infrastructure.md). Splatting and argument forms in the parser: [02.2-parser](../02-language/02.2-parser.md). Conversion rules: [CONVERSION-RULES](../02-language/CONVERSION-RULES.md). + +## 1. Purpose and scope [SPEC] + +Parameter binding connects the arguments written after a command name (and the objects arriving on the pipeline) to the declared parameters of a cmdlet, function or script. It decides which *parameter set* applies, converts and validates values, supplies defaults, prompts for mandatory values, and reports errors. Native (external) programs have their own argument builder (see [command-discovery-and-invocation](command-discovery-and-invocation.md) §7). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController@L19-L1325 | abstract class | named, positional and splatted binding; argument re-parsing | A | +| src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController@L22-L4270 | class | the full algorithm: parameter sets, defaults, dynamic parameters, mandatory, pipeline binding | A | +| src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase@L57-L1937 | abstract class | binds one value: transformation, coercion, validation, set | A | +| src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata@L12-L637 | class | all bindable parameters of a command (declared, common, dynamic) with prefix matching and set bitmasks | A | +| src/System.Management.Automation/engine/CompiledCommandParameter.cs::CompiledCommandParameter@L15-L568 | class | one parameter's compiled metadata | A | +| src/System.Management.Automation/engine/ParameterSetSpecificMetadata.cs::ParameterSetSpecificMetadata@L6-L213 | class | per-set facts: position, mandatory, pipeline flags | B | +| src/System.Management.Automation/engine/TypeMetadata.cs::InternalParameterMetadata@L1117-L1566 | class | builds parameter metadata from a type or a script block | B | +| src/System.Management.Automation/engine/scriptparameterbindercontroller.cs::ScriptParameterBinderController@L14-L230 | class | script/function variant | A | + +## 3. Responsibilities and non-responsibilities + +Owns the order of binding steps, parameter-set selection, mandatory handling, pipeline-input binding order, and the per-value conversion/validation order. Does not own: type conversion rules (`LanguagePrimitives`, T28), the attribute classes themselves, the cmdlet lifecycle (T31), `$PSDefaultParameterValues` storage (a variable), or parsing of `-name:value` forms (parser). + +## 4. Key types and contracts [IMPL] + +- `MergedCompiledCommandParameter` = a parameter plus its *binder association* (`DeclaredFormalParameters`, `DynamicParameters`, common/ShouldProcess/paging/transaction parameter binders). `MergedCommandParameterMetadata` keeps two maps: bindable names and alias names; adding a name or alias that already exists raises a `MetadataException` (`ParameterNameAlreadyExistsForCommand`, `ParameterNameConflictsWithAlias`, `AliasParameterNameAlreadyExistsForCommand`) (src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.AddMetadataForBinder@L78-L157). +- **Parameter sets are bit flags**: each distinct set name gets one bit of a 32-bit mask (`MaxParameterSetCount = 32`); a parameter in `__AllParameterSets` has flag 0 and `IsInAllSets`; the default set name (from `DefaultParameterSetName`) is added first so it gets bit 0 (src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.GenerateParameterSetMappingFromMetadata@L278-L341). More than 32 sets is an error at metadata construction (`UNVERIFIED` which error: `AddParameterSetToMap` throws when the index reaches 32). +- `ParameterSetSpecificMetadata` per (parameter, set): `Position` (`int.MinValue` = not positional), `IsMandatory`, `ValueFromPipeline`, `ValueFromPipelineByPropertyName`, `ValueFromRemainingArguments`, `ParameterSetFlag`, `IsInAllSets` (used throughout the controller). +- `ParameterBindingFlags`: `ShouldCoerceType`, `IsDefaultValue`, `DelayBindScriptBlock`, `ThrowOnParameterNotFound`, `None`. +- `GetMatchingParameter(name, throwOnNotFound, tryExactMatching, invocationInfo)`: strips one leading dash; collects every parameter name and alias for which the typed text is a **case-insensitive prefix** (invariant culture); with `tryExactMatching`, an exact (case-insensitive) match returns at once; several candidates are narrowed to declared or dynamic parameters (not common ones) when exactly one remains; otherwise `AmbiguousParameter` is thrown listing the candidates; none → `NamedParameterNotFound` only if asked to throw (src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.GetMatchingParameter@L445-L578). + +## 5. Data model + +The controller holds: `BindableParameters` (merged metadata), `UnboundParameters` (list), `BoundParameters` (name → parameter), `BoundArguments`, `UnboundArguments` (remaining command-line arguments), `CommandLineParameters` (what `$PSBoundParameters` exposes), `_currentParameterSetFlag` (bitmask of sets still possible; starts as `uint.MaxValue`), `_prePipelineProcessingParameterSetFlags` (saved at the end of command-line binding and restored for each pipeline object), `_delayBindScriptBlocks`, `ParametersBoundThroughPipelineInput`, `BoundDefaultParameters`, `DefaultParameterBindingInUse`, `_allDefaultParameterValuePairs`, and the dynamic-parameter binder. `CommandParameterInternal` is the internal form of one argument: optional parameter name text, optional argument value/AST, flags `FromHashtableSplatting`, `SpaceAfterParameter`. + +## 6. Control flow + +```mermaid +flowchart TD + A[arguments from the call site] --> I[InitUnboundArguments: splatted entries moved last] + I --> R[ReparseUnboundArguments: join name-only args with next value] + R --> N[BindNamedParameters] + N --> P[BindPositionalParameters] + P --> D1[ApplyDefaultParameterBinding after positional] + D1 --> V1[ValidateParameterSets] + V1 --> DY[HandleCommandLineDynamicParameters then named and positional again] + DY --> D2[ApplyDefaultParameterBinding after dynamic] + D2 --> RM[HandleRemainingArguments] + RM --> VA[VerifyArgumentsProcessed] + VA --> VS[ValidateParameterSets final] + VS --> M[HandleUnboundMandatoryParameters: prompt or fail] + M --> SB[BindUnboundScriptParameters: defaults for script params] + SB --> W[save pre-pipeline set flags] + W --> PL[per pipeline object: BindPipelineParameters] +``` + +**Parameter binding order and parameter-set resolution (CF-BND-001).** Command-line binding runs these steps in order: (1) arguments that came from hashtable splatting are moved after all other arguments; (2) a parameter name with no value is joined with the next argument as its value (a switch parameter takes `$true` instead; a name followed by another recognizable parameter name, or a name at the end, is a `MissingArgument` error); (3) **named binding**: every argument with a name is matched (exact name or alias first, then unambiguous prefix); a parameter already bound is an error (`ParameterAlreadyBound`) except that a splatted entry whose name was already bound explicitly is skipped; the `--%` verbatim marker is consumed here; (4) **positional binding**: unbound parameters that have a position in the still-valid sets are grouped by position number in ascending order; arguments without a name are taken in order; each is offered to the parameters of that position in four attempts — default set without type coercion, all valid sets without coercion, default set with coercion, all valid sets with coercion — and an argument no parameter accepts stays unbound; (5) values from `$PSDefaultParameterValues` are bound to still-unbound parameters, after positional binding and again after dynamic parameters; (6) `ValidateParameterSets`; (7) dynamic parameters are fetched once from the command (`GetDynamicParameters`), merged into the parameter table and set bitmasks, and named and positional binding are repeated for the leftover arguments; (8) leftover arguments are bound to the single `ValueFromRemainingArguments` parameter (two such parameters in the current sets is an error); (9) any argument still unbound is an error (`NamedParameterNotFound` for a named one, `PositionalParameterNotFound` otherwise); (10) the final parameter-set check and the mandatory-parameter step (§7); (11) script parameters that are still unbound get their declared default values; (12) the set mask is saved for pipeline binding. **Parameter-set resolution:** each bound parameter narrows the current set mask to the sets it belongs to. After binding, with *one* valid set that set is chosen. With *several* valid sets: if the mask is "all sets" and the command has no default set, the call is accepted as set-less; else if the command defines a default set and it is among the valid sets, the command's parameter-set name is set to the default set, and the mask is narrowed to the default set only in validation calls that ask for it (the final check after command-line binding does; the checks made before dynamic parameters and during pipeline binding may not); else, if pipeline input is still expected and some valid set has an unbound parameter that takes pipeline input, the decision is postponed to pipeline binding; else the mask is reduced to sets with no unbound mandatory parameter; if exactly one set remains it is chosen, otherwise `AmbiguousParameterSet`. With *zero* valid sets (conflicting parameters) the result is `AmbiguousParameterSet` as well. **Pipeline-object binding** for each input object restores the saved mask and the original values of parameters bound by the previous object, then makes four passes over the unbound parameters that accept pipeline input — (a) by value without coercion, (b) by property name without coercion, (c) by value with coercion, (d) by property name with coercion — trying the default set before the other valid sets in each pass; a successful bind narrows the set mask again. + +**Binding one value** (src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L224-L224): (1) argument-transformation attributes run in declaration order — the internal type-converter attribute only when coercion is allowed; other transformation attributes run for non-null values, and for null values only if the parameter is mandatory in some set, cannot be null, or the attribute asks for it (and the value is not a default); a failure becomes `ParameterBindingArgumentTransformationException`; (2) if coercion is allowed the value is converted to the parameter type (collections are encoded element-wise), else a non-coercing check decides whether to continue; (3) a `PSTypeName` constraint is validated for each element; (4) unless the value is a default value, every validation attribute runs in order (`ParameterBindingValidationException` on failure), then a null/empty check for mandatory parameters; (5) an obsolete script parameter (non-cmdlet-binding scripts) emits a warning; (6) the value is set on the target (property/field or runtime-defined parameter) and recorded in `$PSBoundParameters`; errors from the setter become `ParameterBindingFailed`. + +**Delay-bind script blocks**: during command-line binding, if an argument is a script block for a parameter that accepts pipeline input and whose type is not itself script-block-compatible, binding is postponed; the parameter counts as bound for set selection, and for every pipeline object the script block is invoked with the object as `$_` and its result is bound (`ScriptBlockArgumentNoInput` if the command can receive no pipeline input at all) (src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindParameter@L1198-L1317). + +## 7. Algorithms + +**Mandatory handling** (src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.HandleUnboundMandatoryParameters@L2780-L2788, `GetMissingMandatoryParameters`): after command-line binding, collect unbound parameters that are mandatory in the current sets. Without expected pipeline input every such parameter is missing. With pipeline input expected, parameters that accept pipeline input (by value or property name) are *not* reported — they may still be bound per object — but non-pipeline mandatory parameters of the chosen set are. Complex rules decide which set to latch onto when the default set and another set both remain (they prefer the default set unless another set takes pipeline input without non-pipeline mandatory parameters); the read part of that function (lines 1756–2000 of 2393) is described here, the remainder is `UNVERIFIED` (VB-030). If any are missing: when the command is public and the host can prompt, the host is asked for each (field descriptions with per-parameter help, element-wise for arrays) and the answers are bound with coercion; otherwise `MissingMandatoryParameter` is thrown listing the names. During pipeline binding, `Read()` calls the same check per object (with `processMissingMandatory = false`) and writes a non-terminating `InputObjectMissingMandatory` error instead (see CF-LCY-001). + +**Default parameter values** (src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindDefaultParameters@L390-L465): pairs from `$PSDefaultParameterValues` qualified by command name patterns apply only to parameters not yet bound and valid in the current sets; a script-block value is evaluated with a state object; zero results skip, one result is used, several results become a collection; values are bound with coercion and delay-bind allowed; a failure produces a one-time warning per command/parameter and is otherwise ignored; bound defaults are recorded in `BoundDefaultParameters`. `UNVERIFIED` (VB-031): the key matching rules in `GetDefaultParameterValuePairs` (lines 591–846). + +**Pipeline binding detail** (src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindPipelineParametersPrivate@L3255-L3347): scanning order inside a pass is from the *last* unbound parameter to the first; by-property-name binding looks up a property of the input object with the parameter's name, then each alias, in that order; errors of kinds transformation, validation or default-value are rethrown (elaborated when default binding is in use), plain binding failures just mean "not bound"; a `PSInvalidCastException` during the non-coercing pass is not an error. If a set was prioritized (the default set when another set takes pipeline input), the whole four-pass sequence runs first restricted to that set and then, if not yet decided, to the others. + +## 8. Observable behavior [SPEC] + +- Parameter names can be abbreviated to any unambiguous prefix; aliases take part in the prefix match; `-name:value` and `-name value` are equivalent; a switch needs no value. +- Positional arguments fill positions in increasing order, only among parameters of still-possible sets. +- Splatting a hashtable cannot override a parameter bound explicitly. +- A command with several parameter sets and no way to decide fails with "Parameter set cannot be resolved using the specified named parameters"-style `AmbiguousParameterSet`. +- A mandatory parameter that is missing is prompted for (public commands, interactive host) or fails. +- Pipeline input binds by value first, then by property name, exact type before coerced. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`ParameterBindingException` family: `ParameterBindingArgumentTransformationException`, `ParameterBindingValidationException` (has `SwallowException` for positional trials), `ParameterBindingParameterDefaultValueException`. Error ids seen: `MissingArgument`, `ParameterAlreadyBound`, `AmbiguousParameter`, `NamedParameterNotFound`, `PositionalParameterNotFound`, `AmbiguousPositionalParameterNoName`, `AmbiguousParameterSet`, `MissingMandatoryParameter`, `ScriptBlockArgumentNoInput`, `GetDynamicParametersException`, `ParameterArgumentTransformationError`, `ParameterArgumentValidationError`, `ParameterBindingFailed`, `InputObjectNotBound`, `InputObjectMissingMandatory`. When default-parameter binding was used, binding errors are re-thrown as an "elaborated" exception that names the default parameter source (`ThrowElaboratedBindingException`). In the non-coercing positional/pipeline passes a failure whose cause chain contains a `PSInvalidCastException` is swallowed (the coercing pass will retry); other exceptions rethrow. Binding failures at pipeline time are non-terminating per object; failures at command-line time are terminating. + +## 10. Concurrency and threading + +Binding runs on the pipeline thread inside `DoPrepare` (command line) and `Read()` (per object); it uses the command's own controller; nothing is shared across threads. The language mode may change temporarily during binding of a script command (see [runspace-and-pipeline](runspace-and-pipeline.md) §14). + +## 11. Extension points and customization hooks + +Attributes: `Parameter`, `Alias`, `ArgumentTransformation` (user-derivable), `ValidateArgumentsAttribute` family, `PSTypeName`, `Obsolete`, `AllowNull`/`AllowEmptyString`/`AllowEmptyCollection`; `IDynamicParameters.GetDynamicParameters()` returning a `RuntimeDefinedParameterDictionary` or an object with attributed members; `$PSDefaultParameterValues`; script-block arguments for pipeline-bindable parameters; `DefaultParameterSetName` and `ConfirmImpact` in `CmdletBinding`. + +## 12. Dependencies + +Inbound: `CommandProcessor.Prepare` and `Read`. Outbound: `LanguagePrimitives` conversion (T28), `PSObject`/ETS property lookup (T35) for by-property-name, the host UI for prompting, `MshCommandRuntime` (warnings, pipeline-input expectation), the script `ScriptBlock` invoke for delay-bind and default script blocks, `$PSBoundParameters` support in `PSBoundParametersDictionary`. + +## 13. Performance notes + +Metadata is compiled once per command info and cached (`CommandMetadata`); per call the controller allocates sets and lists. Pipeline binding repeats up to four passes per object, so commands with many pipeline-bindable parameters pay per object; the per-object restore of default values (`RestoreDefaultParameterValues`) re-runs for parameters bound by the previous object. + +## 14. Security considerations + +Binding is a policy-relevant boundary: (a) argument-transformation attributes and `ValidateScript`/`ArgumentCompleter` run user code during binding (the language-mode switch for script commands and the `LanguageModeTransitionInParameterBinding` flag exist so that code running there cannot be elevated or leak trusted data) — see src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Prepare@L193-L254; (b) prompting for mandatory parameters is allowed only for commands whose visibility is public; (c) obsolete/deprecated warnings and `LogPipelineExecutionDetail` record bound values (truncated to about 256 characters per value) into logs when module/pipeline logging is on, which is a data-leak channel for secrets bound as strings (`SecureString`/credential types are the mitigation, `INFERENCE`). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 8 | +| **total** | 8 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/ParameterBinderBase.cs` | 8 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: (1) transformation and validation callbacks run with the *callee's* trust level or lower, never higher, and cannot observe unbound trusted values; (2) logging of bound values is redacted by parameter type/attribute; (3) binding never changes the active language mode for code outside the binding call; (4) a binder cannot be used to write to members other than declared parameters. + +## 15. Enterprise considerations + +Security first: §14. Observability: a binding tracer (`bindingTracer`, per-step scoped messages) exists for diagnostics (`Trace-Command ParameterBinding`) and pipeline-detail logging records each bound parameter. Supportability: error ids and `TargetObject`/extent make binding errors precise. Testability: binding is observable end to end through the language; a unit-testable "static" binder exists for completion and analysis (`StaticParameterBinder`, `UNVERIFIED` here). Resource limits: 32 parameter sets per command; no limit on argument count. Missing: a dry-run "explain binding" result usable by tooling without a tracer. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Parser/ParameterBinding.Tests.ps1` | 3 | 0 | 14 | +| `test/powershell/Language/Scripting/CommonParameters.Tests.ps1` | 1 | 10 | 22 | +| `test/powershell/Language/Scripting/Dynamicparameters.Tests.ps1` | 1 | 0 | 5 | +| `test/powershell/Language/Scripting/ParameterBinding.Tests.ps1` | 1 | 2 | 40 | +| `test/powershell/engine/ParameterBinding/BooleanParameterDCR.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/ParameterBinding/NullableBooleanDCR.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/ParameterBinding/ParameterBinding.Tests.ps1` | 1 | 3 | 24 | +| `test/powershell/engine/ParameterBinding/Splatting.Tests.ps1` | 1 | 2 | 11 | +| `test/powershell/engine/ParameterBinding/StaticParameterBinder.Tests.ps1` | 1 | 0 | 1 | +| **total (9 files)** | 11 | 17 | 123 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `engine/ParameterBinding/`, `Parser/ParameterBinding`, `Scripting/ParameterBinding`, `Scripting/CommonParameters`, `Dynamicparameters`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: bitmask parameter sets give constant-time set intersection at the cost of a hard limit of 32 sets. +- `FACT-SRC`: two-stage typed attempts (without, then with coercion) make exact-type matches win over convertible ones for positional and pipeline binding. +- `INFERENCE`: restoring parameters per pipeline object (instead of re-creating the binder) trades a restore pass for lower allocation. +- `FACT-SRC`: default-parameter binding is spread over several points of the sequence (after positional, after dynamic, after pipeline) because dynamic parameters and pipeline sets change which defaults are valid. +- `FACT-SRC`: the controller carries much repeated "default binding in use → elaborate exception" logic; this is a readability cost. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTBIND-01** [REPLICATE] (area: NLD-CMD) Keep the binding order (named → positional → defaults → dynamic → remaining → mandatory → pipeline) and the exact → coerced two-pass for positional and pipeline binding; it is observable. Evidence: src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindCommandLineParametersNoValidation@L202-L277. +- **LSN-RTBIND-02** [REPLICATE] (area: NLD-CMD) Parameter sets as bit masks with a default set and the ambiguity-by-mandatory tie-break. Evidence: src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.ValidateParameterSets@L2532-L2640. +- **LSN-RTBIND-03** [REPLICATE] (area: NLD-CMD) Unambiguous-prefix parameter names including aliases; exact match wins. Evidence: src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.GetMatchingParameter@L445-L578. +- **LSN-RTBIND-04** [ADAPT] (area: NLD-CMD) Lift the 32-set limit by using a set-index bit vector type, keeping the same semantics. Evidence: src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.MaxParameterSetCount@L171-L171. +- **LSN-RTBIND-05** [ADAPT] (area: NLD-CMD) Factor the "default binding in use → elaborate error" pattern into one wrapper, and the binding steps into an explicit pipeline of stages with a trace result object. Evidence: src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController.ThrowElaboratedBindingException@L1017-L1049. +- **LSN-RTBIND-06** [REPLICATE] (area: NLD-SEC) Run transformation/validation callbacks under the callee's language mode; keep the explicit transition flag. Evidence: src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Prepare@L193-L254. +- **LSN-RTBIND-07** [REDESIGN] (area: NLD-SEC) Redact bound values in execution-detail logging by parameter type and attribute instead of logging the first 256 characters of every value. DEV: DEV-BIND-130. Evidence: src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L329-L667. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-030): `GetMissingMandatoryParameters` lines 2000–2393 and `NewParameterSetPromptingData`; `PromptForMissingMandatoryParameters`/`BuildLabel` (prompt UI); `CoerceTypeAsNeeded` and `EncodeCollection` (`ParameterBinderBase` 984–1900; collection encoding rules). +- `UNVERIFIED` (VB-031): `GetDefaultParameterValuePairs` key-matching rules; `RestoreDefaultParameterValues`; `InvokeAndBindDelayBindScriptBlock` details. +- `UNVERIFIED` (VB-032): how `InternalParameterMetadata` builds metadata from attributes and script blocks (limits, error ids), `ScriptParameterBinder`, `ReflectionParameterBinder`, common-parameter binders (`-Verbose`, `-ErrorAction`, …). + +Verification: all non-comment lines of `ParameterBinderController.cs` lines 133–330, 514–870, 919–1190, `CmdletParameterBinderController.cs` lines 104–470, 854–990, 1198–1320, 1463–1580, 1595–1735, 1756–2000, 2532–2700, 2752–2905, 3147–3630, `ParameterBinderBase.cs` lines 329–667 and `MergedCommandParameterMetadata.cs` lines 78–160, 278–345, 445–580 were read. Ranges are in `T31_T39.csv`. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/03-runtime/provider-model.md b/docs-out/03-runtime/provider-model.md new file mode 100644 index 0000000..a9d08e9 --- /dev/null +++ b/docs-out/03-runtime/provider-model.md @@ -0,0 +1,198 @@ +--- +id: RT-PROVIDER-001 +title: Provider model +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/namespaces/ProviderBase.cs, src/System.Management.Automation/namespaces/DriveProviderBase.cs, src/System.Management.Automation/namespaces/ItemProviderBase.cs, src/System.Management.Automation/namespaces/ContainerProviderBase.cs, src/System.Management.Automation/namespaces/NavigationProviderBase.cs, src/System.Management.Automation/namespaces/LocationGlobber.cs, src/System.Management.Automation/namespaces/ProviderDeclarationAttribute.cs, src/System.Management.Automation/engine/SessionStateProviderAPIs.cs, src/System.Management.Automation/engine/SessionStateDriveAPIs.cs, src/System.Management.Automation/engine/DataStoreAdapterProvider.cs, src/System.Management.Automation/engine/DataStoreAdapter.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/namespaces/_folder.md, src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json, data/resources.json] +--- + +# Provider model + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope: `EXTENDED` (providers are not part of the language core). Scopes and drives as language-visible concepts: [SCOPE-AND-VARIABLES](../02-language/SCOPE-AND-VARIABLES.md); the cmdlets that use providers (`Get-ChildItem` etc.) belong to the command-module families (Phase 4). + +## 1. Purpose and scope [SPEC] + +A *provider* exposes a hierarchical data store (file system, registry, variables, aliases, functions, environment, certificates, WSMan configuration) through one uniform set of verbs, so that the same cmdlets work on all of them. A provider is mounted on one or more *drives* (`C:`, `HKLM:`, `Env:`, `Variable:`); a *path* names an item within a provider. This document describes the provider class hierarchy and contracts, the drive and provider tables, and path resolution (drive-qualified, provider-qualified, relative, globbed). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/namespaces/ProviderBase.cs::CmdletProvider@L54-L1981 | abstract partial class | base: `Start`/`Stop`, context-bound properties, stream-writing API, `ShouldProcess` | B | +| src/System.Management.Automation/namespaces/DriveProviderBase.cs::DriveCmdletProvider@L20-L235 | abstract class | `NewDrive`, `RemoveDrive`, `InitializeDefaultDrives` | B | +| src/System.Management.Automation/namespaces/ItemProviderBase.cs::ItemCmdletProvider@L22-L687 | abstract class | `GetItem`, `SetItem`, `ClearItem`, `InvokeDefaultAction`, `ItemExists`, `IsValidPath`, `ExpandPath` | B | +| src/System.Management.Automation/namespaces/ContainerProviderBase.cs::ContainerCmdletProvider@L30-L1083 | abstract class | children, names, rename, new, remove, copy, `HasChildItems`, `ConvertPath` | B | +| src/System.Management.Automation/namespaces/NavigationProviderBase.cs::NavigationCmdletProvider@L21-L1088 | abstract class | `MakePath`, `GetParentPath`, `NormalizeRelativePath`, `GetChildName`, `IsItemContainer`, `MoveItem` | B | +| src/System.Management.Automation/namespaces/IContentProvider.cs::IContentCmdletProvider@L29-L146 | interface | content reader/writer/clear | B | +| src/System.Management.Automation/namespaces/IPropertiesProvider.cs::IPropertyCmdletProvider@L29-L198 | interface | item properties | B | +| src/System.Management.Automation/namespaces/IDynamicPropertyProvider.cs::IDynamicPropertyCmdletProvider@L28-L336 | interface | create/rename/copy/move/remove properties | B | +| src/System.Management.Automation/namespaces/IPermissionProvider.cs::ISecurityDescriptorCmdletProvider@L24-L107 | interface | security descriptors | B | +| src/System.Management.Automation/namespaces/LocationGlobber.cs::LocationGlobber@L17-L4731 | class | path forms, globbing, drive and provider resolution | B | +| src/System.Management.Automation/namespaces/CoreCommandContext.cs::CmdletProviderContext@L24-L1147 | sealed class | per-operation context: stream sink, filters, credential, force, drive | B | +| src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo@L22-L687 | class | description of a registered provider; creates instances | B | +| src/System.Management.Automation/engine/DataStoreAdapter.cs::PSDriveInfo@L22-L831 | class | one mounted drive | B | +| src/System.Management.Automation/engine/SessionStateProviderAPIs.cs, SessionStateDriveAPIs.cs | partial `SessionStateInternal` | provider table and per-scope drive tables | B | +| src/System.Management.Automation/namespaces/FileSystemProvider.cs, RegistryProvider.cs, AliasProvider.cs, FunctionProvider.cs, VariableProvider.cs, EnvironmentProvider.cs | provider classes | built-in providers | B | +| src/System.Management.Automation/namespaces/SessionStateProviderBase.cs | abstract class | shared base of the alias/function/variable/environment providers | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the provider class contracts, provider and drive registration, path-to-provider resolution, per-operation context. Does not own: the command-line surface (`*-Item`, `*-ChildItem`, … cmdlets), format views of items (T36), session-state scope mechanics (T29), the implementations of individual providers beyond their declaration. + +## 4. Key types and contracts [IMPL] + +- **Hierarchy**: `CmdletProvider` → `DriveCmdletProvider` → `ItemCmdletProvider` → `ContainerCmdletProvider` → `NavigationCmdletProvider`. A provider class derives from the deepest level it supports; optional capabilities are interfaces (`IContentCmdletProvider`, `IPropertyCmdletProvider`, `IDynamicPropertyCmdletProvider`, `ISecurityDescriptorCmdletProvider`). The engine obtains the needed level with `GetDriveProviderInstance`, `GetItemProviderInstance`, `GetContainerProviderInstance`, `GetNavigationProviderInstance`, each of which fails when the provider does not derive from that level. +- **Declaration**: `[CmdletProvider(name, ProviderCapabilities)]`. `ProviderCapabilities` flags: `None=0`, `Include=1`, `Exclude=2`, `Filter=4`, `ExpandWildcards=8`, `ShouldProcess=0x10`, `Credentials=0x20`, `Transactions=0x40` (src/System.Management.Automation/namespaces/ProviderDeclarationAttribute.cs). Built-in declarations: FileSystem (`Credentials | Filter | ShouldProcess`), Alias, Function, Variable, Environment (`ShouldProcess`), Registry (`ShouldProcess`, plus `Transactions` on Windows), Certificate (`ShouldProcess`), WSMan (`Credentials`). +- **Virtual surface** (protected, empty or throwing by default): Drive level `NewDrive`, `RemoveDrive`, `InitializeDefaultDrives`, `NewDriveDynamicParameters`; Item level `GetItem`, `SetItem`, `ClearItem`, `InvokeDefaultAction`, `ItemExists`, `ExpandPath`, abstract `IsValidPath`, plus a `…DynamicParameters` twin for each; Container level `GetChildItems(path, recurse[, depth])`, `GetChildNames`, `ConvertPath`, `RenameItem`, `NewItem`, `RemoveItem`, `HasChildItems`, `CopyItem`; Navigation level `MakePath`, `GetParentPath`, `NormalizeRelativePath`, `GetChildName`, `IsItemContainer`, `MoveItem`; base level `Start`, `StartDynamicParameters`, `Stop`, `ItemSeparator`, `AltItemSeparator`, `GetResourceString`. +- **Internal call pattern**: every engine call goes through an *internal* method that stores the `CmdletProviderContext` into the provider's `Context` and then calls the protected virtual (`GetItem(path, context)` → `Context = context; GetItem(path)`). `ItemExists` additionally catches every exception and returns false (src/System.Management.Automation/namespaces/ItemProviderBase.cs::ItemCmdletProvider.ItemExists@L248-L266). +- **Output from providers** is written through `WriteItemObject(item, path, isContainer)`, `WritePropertyObject`, `WriteSecurityDescriptorObject` and `WriteError`, which forward to the context (and so to the calling cmdlet's pipeline); `ShouldProcess`/`ShouldContinue` forward to the calling cmdlet's runtime. The context-bound properties `Credential`, `Filter`, `Include`, `Exclude`, `Force`, `DynamicParameters`, `PSDriveInfo`, `ProviderInfo`, `Stopping`, `Host` and `SessionState` are read from the current context. +- **A new provider instance per operation**: `ProviderInfo.CreateInstance` calls `Activator.CreateInstance` on the implementing type every time; the comment in source explains that thread-local caching was dropped because a shared instance's `Context` gets overwritten when two commands using the same provider are piped together (src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo.CreateInstance@L546-L638). Provider classes must therefore not keep state between calls (state lives in the drive object or the store). + +## 5. Data model + +`ProviderInfo`: name, implementing type, capabilities, home, drives, module/snap-in, help file, `ItemSeparator`. `PSDriveInfo`: name, provider, root, current location, description, credential, `IsAutoMounted`, display root, scope-specific. Session state holds a provider table (name → list of `ProviderInfo`; the same name can exist in several modules, and a lookup that matches more than one raises `ProviderNameAmbiguousException`), per-scope drive tables, and `ProvidersCurrentWorkingDrive` (provider → current drive). Path forms (from `LocationGlobber`): *drive-qualified* `name:path`; *provider-qualified* `provider::path` (detected when a `::` follows the first colon, the text before it is non-empty and the path does not start with `.\` or `./`); *provider-internal* path (what the provider itself understands); *relative* (resolved against the current location of the current drive); *home* (`~`); *absolute* if it starts with a single file-system absolute form or contains a drive name before any path separator. + +## 6. Control flow + +```mermaid +flowchart TD + P[PS path typed by user] --> Q{provider-qualified?} + Q -- yes --> PI[provider id and provider-internal path] + Q -- no --> D{drive-qualified or absolute?} + D -- yes --> DR[find drive in scope chain, else auto-mount] + D -- no --> CUR[current location of current drive] + PI --> G[globbing if wildcard characters and provider allows] + DR --> G + CUR --> G + G --> PP[provider-internal paths] + PP --> CALL[internal provider method with CmdletProviderContext] + CALL --> OUT[WriteItemObject etc to calling pipeline] +``` + +**Drive lookup** (src/System.Management.Automation/engine/SessionStateDriveAPIs.cs::SessionStateInternal.GetDrive@L415-L477): search the current scope, then each parent scope; an auto-mounted drive found is first validated and dropped if stale; if none is found, try to *auto-mount* a file-system drive for that name (and then a built-in default drive); else `DriveNotFoundException`. + +**Provider initialization** (`InitializeProvider`): ask the drive-level provider for its default drives; the first becomes the provider's current working drive; each drive whose provider name matches is validated with the provider (`NewDrive`) and added to the **global scope**; a failure writes an error rather than aborting. A new drive (`New-PSDrive`) is validated by the provider, must have a legal name, may keep a different name than requested only if names match case-insensitively (else `NewDriveProviderFailed`), is added to the current or named scope, and becomes the provider's current drive if none is set. + +## 7. Algorithms + +Path resolution from a PS path to provider paths: `GetGlobbedProviderPathsFromMonadPath(path, allowNonexistingPaths, context, out provider, out providerInstance)`: if the path is provider-qualified the context's drive is cleared; the path is resolved to a provider path and drive (`GetProviderPath`); if the provider returns no path the result is empty; the drive is placed in the context; the PS path is globbed (`GetGlobbedMonadPathsFromMonadPath`) and the provider-internal parts of the results are returned (src/System.Management.Automation/namespaces/LocationGlobber.cs::LocationGlobber.GetGlobbedProviderPathsFromMonadPath@L732-L790). Globbing (expand wildcard characters, generate paths for a glob leaf, match child names) is implemented in `ExpandGlobPath`, `ExpandMshGlobPath`, `GenerateNewPathsWithGlobLeaf` and `IsChildNameAMatch` (`UNVERIFIED` in detail, VB-035); whether a path is globbed at all depends on the provider's capabilities and on the context (`ShouldPerformGlobbing`). + +## 8. Observable behavior [SPEC] + +- `Get-PSProvider` lists providers; `Get-PSDrive` lists drives; `Set-Location Env:` changes the current drive; a path like `HKLM:\Software` is resolved through the Registry provider. +- The same commands act differently per provider, controlled by capabilities (`-Filter` only if the provider declares `Filter`; `-Credential` only with `Credentials`). +- File-system drives for existing volumes are auto-mounted on first use. +- A provider that throws during an operation produces a `ProviderInvocationException` with the provider and path. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`DriveNotFoundException`, `ProviderNotFoundException` (also for provider constructor failures: the constructor's inner exception is attached), `ProviderNameAmbiguousException`, `ItemNotFoundException`, `NotSupportedException`/`PSNotSupportedException` when a provider lacks a level or interface, `ProviderInvocationException` wrapping provider exceptions. During path resolution, `ItemNotFoundException`, `DriveNotFoundException`, `ProviderNotFoundException`, an unset provider home (`InvalidOperationException`), provider errors and "provider does not implement the container level" are all caught when the command searcher resolves a command name through a path (it returns no match) (src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.GetNextFromPathUsingWildcards@L520-L566). + +## 10. Concurrency and threading + +Providers are created per operation and run on the calling pipeline thread; the `Context` field is not thread-safe and is the reason instances are not shared. `Stopping` reflects the calling pipeline's stop state. Content readers/writers returned by providers are used from the calling cmdlet. + +## 11. Extension points and customization hooks + +A new provider class with `[CmdletProvider]`; registration through `InitialSessionState` (`SessionStateProviderEntry`) or module manifests/snap-ins; dynamic parameters per operation (`…DynamicParameters` methods); `ItemSeparator` / `AltItemSeparator`; per-provider resource strings (`GetResourceString`); custom drives from `New-PSDrive -PSProvider` with the provider's `NewDrive`. The default built-in set registered in `InitialSessionState` is Registry (Windows), Alias, Environment, FileSystem, Function, Variable (src/System.Management.Automation/engine/InitialSessionState.cs::PSSnapInHelpers@L5030-L5613 — the table in source); Certificate and WSMan come from their own assemblies. + +## 12. Dependencies + +Inbound: item/location cmdlets, `Resolve-Path`, the command searcher (path lookup), tab completion, `Test-Path`, native argument globbing (`PossiblyGlobArg`, [command-discovery-and-invocation](command-discovery-and-invocation.md)), session-state variable/function/alias lookups (the variable, function and alias providers are views over session state). Outbound: session state (drive tables, provider table), the host (prompts), the file system and registry APIs, `PSObject` (items written as `PSObject` with provider-path properties added; `UNVERIFIED` in detail). + +## 13. Performance notes + +New instance per operation (activator call) is cheap relative to I/O but adds allocation per call; globbing may enumerate directories; auto-mounting scans system drives on a miss; `PSDrive` validation calls the provider for each new drive. + +## 14. Security considerations + +The provider boundary is where scripts reach the host's file system, registry and credentials. Enforced here: provider capabilities gate `-Credential`, `-Filter`; `ShouldProcess` capability routes destructive operations through consent; visibility of providers and drives is a session-state property (restricted runspaces can omit providers). Not enforced here: path allow-listing, sandbox roots (the file-system provider accepts any path the process can access), and symbolic-link policy (`UNVERIFIED`). Credentials flow into the provider as `PSCredential` on the context. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 3 | +| trust-boundary | 2 | +| securestring | 1 | +| **total** | 6 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/namespaces/FileSystemProvider.cs` | 6 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: all item access goes through a mediation point that can apply a capability policy (root confinement, read/write/delete rights) *before* the provider code; provider-supplied paths are normalized by the mediation point, never by provider code alone; credentials on the context are not retained by provider instances. + +## 15. Enterprise considerations + +Security first: §14. Observability: path-resolution and provider-base tracers exist (`s_pathResolutionTracer`, `providerBaseTracer`); provider exceptions carry provider name and path. Supportability: the uniform verbs make third-party stores scriptable. Testability: a provider can be tested through the cmdlets; internal methods are `internal`, so unit tests need friend access. Resource limits: none. Missing: asynchronous provider operations, cancellation tokens (only the `Stopping` flag), per-operation audit events. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Modules/Microsoft.PowerShell.Core/ImportProvider.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Alias.Tests.ps1` | 2 | 1 | 11 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1` | 12 | 26 | 146 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/FileSystemProviderExtended.Tests.ps1` | 2 | 16 | 86 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/FunctionProvider.Tests.ps1` | 1 | 4 | 15 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Get-PSDrive.Tests.ps1` | 3 | 0 | 12 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Get-PSProvider.Tests.ps1` | 1 | 1 | 6 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/New-PSDrive.Tests.ps1` | 1 | 1 | 4 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/PSDrive.Tests.ps1` | 2 | 4 | 14 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Push-Location.Tests.ps1` | 1 | 0 | 9 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Registry.Tests.ps1` | 2 | 13 | 52 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Resolve-Path.Tests.ps1` | 1 | 0 | 8 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Set-Location.Tests.ps1` | 1 | 4 | 23 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/Variable.Tests.ps1` | 2 | 0 | 10 | +| `test/powershell/Modules/Microsoft.PowerShell.Security/CertificateProvider.Tests.ps1` | 2 | 5 | 19 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Environment-Variables.Tests.ps1` | 1 | 1 | 7 | +| **total (16 files)** | 35 | 76 | 423 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_FileSystemProvider.cs` | 9 | 0 | `FileSystemProviderTests` | + +Matching is by path keyword (`APPROX`): `Management/FileSystem.Tests`, `Management/FileSystemProviderExtended`, `Management/Registry.Tests`, `Management/Alias.Tests`, `Management/FunctionProvider`, `Management/Variable.Tests`, `Management/PSDrive.Tests`, `Management/New-PSDrive`, `Management/Get-PSDrive`, `Management/Get-PSProvider`, `Security/CertificateProvider`, `ImportProvider`, `Management/Resolve-Path`, `Management/Set-Location`, `Management/Push-Location`, `Environment-Variables`, `test_FileSystemProvider`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: no provider instance sharing (documented in source). +- `FACT-SRC`: capabilities are declared in an attribute and also expressed by the class level; both must agree. +- `INFERENCE`: `ItemExists` swallowing exceptions makes `Test-Path` robust at the cost of hiding provider faults. +- `FACT-SRC`: drives live in scopes (like variables), so a script can mount a drive visible only to its callees. +- `INFERENCE`: the variable, alias and function stores are exposed as providers so that the same verbs (`Get-ChildItem Variable:`) work on them. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTPROV-01** [ADAPT] (area: NLD-MOD) Keep the verb contract (item, container, navigation, content, property) as interfaces with capability flags instead of a five-level class hierarchy with context-bound state. Evidence: src/System.Management.Automation/namespaces/NavigationProviderBase.cs::NavigationCmdletProvider@L21-L1088. +- **LSN-RTPROV-02** [REPLICATE] (area: NLD-MOD) Drive-qualified, provider-qualified and relative path forms and per-scope drives. Evidence: src/System.Management.Automation/namespaces/LocationGlobber.cs::LocationGlobber.IsProviderQualifiedPath@L1371-L1431. +- **LSN-RTPROV-03** [REDESIGN] (area: NLD-SEC) Put a mandatory policy mediation layer between the engine and providers (root confinement, rights, audit) and pass an explicit per-call context object instead of mutating a `Context` field on the provider. DEV: DEV-PROV-140. Evidence: src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo.CreateInstance@L546-L638. +- **LSN-RTPROV-04** [REPLICATE] (area: NLD-MOD) Treat variables, functions, aliases and environment as providers over session state. Evidence: src/System.Management.Automation/engine/InitialSessionState.cs::PSSnapInHelpers@L5030-L5613. +- **LSN-RTPROV-05** [ADAPT] (area: NLD-MOD) Replace the all-exceptions-swallowed `ItemExists` with a typed result (exists, absent, failed) and let `Test-Path` decide. Evidence: src/System.Management.Automation/namespaces/ItemProviderBase.cs::ItemCmdletProvider.ItemExists@L248-L266. +- **LSN-RTPROV-06** [AVOID] (area: NLD-MOD) Do not implement file-system auto-mounting of Windows volumes on lookup miss as an engine feature; mount drives explicitly or from configuration. DEV: DEV-PROV-141. Evidence: src/System.Management.Automation/engine/SessionStateDriveAPIs.cs::SessionStateInternal.GetDrive@L415-L477. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-035): globbing internals in `LocationGlobber` (`ExpandGlobPath` etc.), `GetProviderPath` variants, home-path rules, `CmdletProviderContext` members, `ProviderIntrinsics` (`InvokeProvider`) facade, `SessionStateContainer`/`SessionStateItem` call routing. +- `UNVERIFIED` (VB-036): `FileSystemProvider` (9,600 lines), `RegistryProvider`, `CertificateProvider`, `WSManConfigProvider` implementations; symbolic-link handling; content stream encodings. +- `UNVERIFIED` (VB-037): how items written by providers are decorated (`PSPath`, `PSParentPath`, `PSChildName`, `PSDrive`, `PSProvider` note properties) — believed from the type-extension data, not read here. + +Verification: `LocationGlobber.cs` lines 732–790, 1348–1445, 1469–1560 and 2380–2454, `ProviderBase.cs` outline and lines 1883–1950, `ItemProviderBase.cs` lines 22–330, `SessionStateProviderAPIs.cs` lines 253–297 and 953–1070, `SessionStateDriveAPIs.cs` lines 64–232 and 410–477, `DataStoreAdapterProvider.cs` lines 546–620 and the `[CmdletProvider]` declarations were read. All other members are from signatures only (tier B). Ranges in `T31_T39.csv`. diff --git a/docs-out/03-runtime/runspace-and-pipeline.md b/docs-out/03-runtime/runspace-and-pipeline.md new file mode 100644 index 0000000..5562184 --- /dev/null +++ b/docs-out/03-runtime/runspace-and-pipeline.md @@ -0,0 +1,268 @@ +--- +id: RT-PIPELINE-001 +title: Runspace and pipeline +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-LCY-001] +sources: [src/System.Management.Automation/engine/pipeline.cs, src/System.Management.Automation/engine/Pipe.cs, src/System.Management.Automation/engine/CommandProcessorBase.cs, src/System.Management.Automation/engine/CommandProcessor.cs, src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs, src/System.Management.Automation/engine/hostifaces/pipelinebase.cs, src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs, src/System.Management.Automation/engine/hostifaces/LocalConnection.cs, src/System.Management.Automation/engine/hostifaces/PowerShell.cs, src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs, src/System.Management.Automation/engine/AutomationEngine.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md, src-mirror/src/System.Management.Automation/engine/hostifaces/_folder.md] +generated_from: [data/members-tierA.json, data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Runspace and pipeline + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Command resolution is in [command-discovery-and-invocation](command-discovery-and-invocation.md); parameter binding in [parameter-binding](parameter-binding.md); the execution context in [execution-context-and-session-state](execution-context-and-session-state.md). The path from host to script is in [LIFECYCLE-script-execution](../02-language/LIFECYCLE-script-execution.md). + +## 1. Purpose and scope [SPEC] + +A *runspace* is one isolated engine instance (one `ExecutionContext`, one session state, one language mode). A *pipeline* is one request to run a list of commands in a runspace. The engine runs a pipeline as a chain of *command processors* joined by *pipes*. This document covers: the three public layers (`PowerShell` API, `Pipeline`, `Runspace`/`RunspacePool`), the engine-internal pipeline (`PipelineProcessor`, `Pipe`, `CommandProcessorBase`, `CommandProcessor`), the state machines, the thread model, stop/cancel, and object flow. Remote runspaces and PSRP are out of scope (Phase 4); only the points where the local code branches on "remote" are named. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor@L29-L1667 | class | engine pipeline: owns the command list, drives begin/process/end, error and stop state | A | +| src/System.Management.Automation/engine/Pipe.cs::Pipe@L29-L621 | class | one connection between two commands (queue or sink) | A | +| src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase@L16-L1043 | abstract class | per-command driver: prepare/begin/execute/complete/cleanup wrappers | A | +| src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor@L21-L868 | class | processor for cmdlets and for advanced functions/scripts | A | +| src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline@L21-L1143 | sealed class | public-layer pipeline that runs on a thread and owns a `PipelineProcessor` | B | +| src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::PipelineStopper@L1266-L1399 | class | stack of active `PipelineProcessor`s; stop fan-out | B | +| src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase@L20-L1071 | abstract class | invoke/stop state machine, streams, events | B | +| src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs::RunspaceBase@L24-L1610 | abstract class | runspace state machine, running-pipeline list, concurrency checks | B | +| src/System.Management.Automation/engine/hostifaces/LocalConnection.cs::LocalRunspace@L31-L1306 | class | local runspace: open creates the engine | B | +| src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs::RunspacePoolInternal@L19-L1668 | class | pool of runspaces (min/max, get/release) | B | +| src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PowerShell@L575-L5954 | sealed class | fluent API over a runspace or pool | B | +| src/System.Management.Automation/engine/AutomationEngine.cs::AutomationEngine@L15-L128 | class | creates the context, parser and command discovery for a runspace | B | + +## 3. Responsibilities and non-responsibilities + +Owns: object flow between commands, ordering of the command lifecycle, the first-error rule, stop propagation, one-pipeline-at-a-time per runspace, runspace/pipeline/`PowerShell` state machines. Does not own: command lookup (T32), argument binding (T33), cmdlet base class behavior and stream-writing APIs (T33), formatting of results (T36), the host UI (Phase 4). + +## 4. Key types and contracts [IMPL] + +- **`Pipe`** is a one-way connection owned by the *upstream* command. Its sink is exactly one of: a queued `ObjectQueue` with a `DownstreamCmdlet`; a nested `PipelineProcessor`; a `Collection`; a `List`; an `ExternalWriter`. `Add` drops `AutomationNull`, appends to the `-OutVariable`-style lists, honours the null pipe, updates the `-PipelineVariable`, then writes to the sink in the order *nested pipeline processor → result collection → result list → external writer → queue* (src/System.Management.Automation/engine/Pipe.cs::Pipe.Add@L406-L425, src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466). `AddItems` enumerates its argument and adds each element; `AutomationNull` elements are skipped (src/System.Management.Automation/engine/Pipe.cs::Pipe.AddItems@L483-L532). +- **Read order of a pipe**: queued objects first, then the wrapped enumerator (skipping `AutomationNull`), then the external reader; end of input is signalled by `AutomationNull.Value`; a closed or disposed external reader is treated as end of input (src/System.Management.Automation/engine/Pipe.cs::Pipe.Retrieve@L541-L601). +- **`PipelineProcessor.Add`** wires a command: the first command takes no input; every later command takes the previous command's output pipe (or its error pipe when `readErrorQueue`), and a pipe may have only one downstream consumer (`PipeAlreadyTaken`). A command that merges unclaimed earlier error results (`MergeUnclaimedPreviousErrorResults`) redirects the error pipes of all earlier commands that have no consumer into its input pipe (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.AddCommand@L331-L440). A command processor can be added once (`CommandProcessorAlreadyUsed`) and not after execution started (`ExecutionAlreadyStarted`). +- **Native chains**: when two native commands are adjacent, the first records the second as `DownStreamNativeCommand` and the second sets `UpstreamIsNativeCommand`, so bytes can pass between processes (the experimental `PSNativeCommandPreserveBytePipe` feature; here only the wiring is read) (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Add@L270-L296). +- **`CommandProcessorBase`** wraps every phase in the same frame: remember the current command processor, switch the engine session state and scope to the command's scope (`SetCurrentScopeToExecutionScope`), run the phase, restore both in `finally` (src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.DoPrepare@L451-L481, src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.DoExecute@L566-L582). `DoBegin` runs `BeginProcessing` once (`RanBeginAlready`); `Complete` first drains remaining input through `ProcessRecord`, then calls `EndProcessing`; `DoCleanup` runs the `clean` block. +- **Write permission**: a command may write to the pipeline only inside `AllowThisCommandToWrite(true)` scopes, which wrap begin, each process call, end and clean; preparation wraps binding in `AllowThisCommandToWrite(false)` (src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.BindCommandLineParameters@L164-L179). +- **`ManageInvocationException`** converts exceptions into the pipeline's stop signal: provider errors become `CmdletProviderInvocationException`; `PipelineStoppedException`, `CmdletInvocationException`, `ActionPreferenceStopException`, `HaltCommandException`, flow-control exceptions, `ScriptCallDepthException` and exceptions from a `throw` statement pass through; any other exception is wrapped in `CmdletInvocationException`; the result is handed to the command runtime (`ManageException`), which returns a `PipelineStoppedException` (src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.ManageInvocationException@L848-L947). +- **`PipelineBase`** keeps the public state (`PipelineState`), the three object streams (input, output, error), and a queue of state events that are raised outside the lock (src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.SetPipelineState@L798-L823, src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.RaisePipelineStateEvents@L837-L890). A state-change handler that throws is swallowed. + +## 5. Data model + +State enumerations (values from source): `RunspaceState` = BeforeOpen, Opening, Opened, Closed, Closing, Broken, Disconnecting, Disconnected, Connecting (src/System.Management.Automation/engine/hostifaces/Connection.cs::RunspaceState@L160-L198); `RunspacePoolState` has the same nine members (src/System.Management.Automation/engine/hostifaces/RunspacePool.cs::RunspacePoolState@L199-L240); `PipelineState` and `PSInvocationState` = NotStarted, Running, Stopping, Stopped, Completed, Failed, Disconnected (src/System.Management.Automation/engine/hostifaces/Pipeline.cs::PipelineState@L132-L162, src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PSInvocationState@L133-L163); `RunspaceAvailability` = None, Available, AvailableForNestedCommand, Busy, RemoteDebug (src/System.Management.Automation/engine/hostifaces/Connection.cs::RunspaceAvailability@L358-L386); `PSThreadOptions` = Default, UseNewThread, ReuseThread, UseCurrentThread (src/System.Management.Automation/engine/hostifaces/Connection.cs::PSThreadOptions@L203-L226). + +`PipelineProcessor` state: the command list, `_executionStarted`, `_stopping`, the first terminating error (`ExceptionDispatchInfo`, `_firstTerminatingError`), optional external input reader, external success/error writers, redirection pipelines, a `CancellationToken` source (`PipelineStopToken`), the execution scope, and an event-log buffer used only when some command asks for pipeline execution logging (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor@L29-L1667). Each command's `MyInvocation` receives `PipelinePosition` (1-based), `PipelineLength`, a shared `PipelineIterationInfo` counter array (element 0 counts input objects of command 1, element *i* counts process calls of command *i*), and `ExpectingInput` (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147, src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.ProcessRecord@L310-L421). + +## 6. Control flow + +```mermaid +sequenceDiagram + participant H as Host or API caller + participant PB as PipelineBase + participant RB as RunspaceBase + participant LP as LocalPipeline + participant PP as PipelineProcessor + participant CP as CommandProcessor + H->>PB: Invoke / InvokeAsync + PB->>RB: DoConcurrentCheckAndAddToRunningPipelines + PB->>PB: SetPipelineState Running + PB->>LP: StartPipelineExecution + LP->>LP: choose thread by PSThreadOptions + LP->>PP: CreatePipelineProcessor then SynchronousExecuteEnumerate + PP->>CP: DoPrepare, then DoBegin for every command + PP->>CP: DoExecute first command + CP->>CP: ProcessRecord loop, Pipe.Add downstream DoExecute + PP->>CP: DoComplete in order (drain, EndProcessing) + PP->>PP: Clean (clean blocks) then DisposeCommands + LP->>PB: SetPipelineState Completed / Failed / Stopped +``` + +**Ordered steps of `PipelineProcessor.SynchronousExecuteEnumerate`** (the top-level run; src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.SynchronousExecuteEnumerate@L493-L590): (1) throw `PipelineStoppedException` if already stopping; (2) `Start`; (3) attach the external input reader to the first command's input pipe; (4) `Inject` the input (an enumerable input is wrapped in a pipe over its enumerator; a scalar is added to the first pipe) and call `DoExecute` on the first command; (5) `DoCompleteCore`; (6) in `finally`, `Clean`; (7) on success, complete redirection pipelines and return the results of the last command; (8) on a `RuntimeException`, rethrow the *first* recorded terminating error; (9) always `DisposeCommands`. + +**`Start`** (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147): checks disposed/stopping/already-started and that at least one command exists; fixes the execution scope to the first command's current scope; connects the external success and error writers; marks the first command's input closed when there is neither external input nor an incoming stream; reads `$PSDefaultParameterValues`; then for every command in order logs "started", fills `MyInvocation` and calls `DoPrepare` (which binds command-line parameters, so **all commands are bound before any begin block runs**); then sets up `-OutVariable`/`-ErrorVariable`/`-WarningVariable`/`-PipelineVariable`/`-InformationVariable` for every command; finally calls `DoBegin` for every command in order. + +**Command lifecycle (CF-LCY-001).** For a pipeline of commands C1…Cn the engine (1) builds all command processors and wires pipes; (2) *prepares* every command in order (parameter binding from the command line, with default parameter values applied) and sets up common-parameter variables; (3) runs the *begin* phase of every command in order C1…Cn (`BeginProcessing`, or the `begin` block); (4) runs the *process* phase *depth-first and synchronously*: C1 reads its input (or runs once with no input if none is expected), and each object it writes into its output pipe causes C2 to run its process phase over everything queued (the default buffer threshold is zero), recursively; every input object is bound before each call (pipeline parameter binding, then mandatory-parameter check); an object that cannot be bound or misses a mandatory parameter produces a non-terminating error and is skipped; (5) when C1's input is exhausted the *end* phase runs for C1…Cn in order — each command first drains any input still queued through its process phase, then runs `EndProcessing` (or the `end` block); (6) the *clean* phase (`clean` blocks, only for commands that have one) runs for every command in `finally`, with stopping suspended, and its errors are written to the command's error output; (7) all command processors are disposed. The first terminating error is recorded; later failures are logged as health events and do not replace it. A stop request cancels the pipeline token and calls `StopProcessing` on every command. + +**Object flow and back-pressure.** Within one pipeline the flow is a synchronous push: `Pipe.AddToPipe` enqueues, and when the downstream command exists and the queue length exceeds `OutBufferCount` (default 0) it calls the downstream command's `DoExecute` on the same thread, so the producer is suspended while the consumer runs (src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466, src/System.Management.Automation/engine/Pipe.cs::Pipe.OutBufferCount@L110-L110). There is no queue limit between commands and no blocking; memory is bounded by the producer's batch, not by a limit. The only bounded blocking queue is the host-facing stream: `ObjectStream` has a capacity (default `Int32.MaxValue`) and writers block on a write handle when it is full (src/System.Management.Automation/utils/ObjectStream.cs::ObjectStream@L395-L1517; the capacity chosen by `PipelineBase` for its three streams is `UNVERIFIED`, VB-024). + +## 7. Algorithms + +**Stop request** (`UNVERIFIED` only where marked). `PipelineBase.CoreStop` (src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.CoreStop@L287-L351): under the lock, `NotStarted` → `Stopping` then `Stopped` and return; `Stopped`/`Completed`/`Failed` → return; `Stopping` → wait for the finished event if synchronous; `Running` → `Stopping`. Then raise state events; if the state is already `Stopped`, return; else `ImplementStop`. `LocalPipeline.StopHelper` (src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.StopHelper@L773-L798): release the debugger; stop nested pipelines started after this one; close the input stream; call `PipelineStopper.Stop`; wait for the pipeline-finished event. `PipelineStopper.Stop` marks itself stopping (a second call is a no-op) and calls `PipelineProcessor.Stop` on every stacked processor. `PipelineProcessor.Stop` records `PipelineStoppedException` as the first error (returns if the pipeline was already stopping), cancels `PipelineStopToken`, and calls `DoStopProcessing` on each command, ignoring exceptions from each (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Stop@L890-L927). An asynchronous stop runs the same helper on a new thread. + +**Early termination by a downstream command** (e.g. `Select-Object -First`): a command throws `StopUpstreamCommandsException`; `SynchronousExecuteEnumerate` and `DoCompleteCore` catch the resulting `PipelineStoppedException` when the first error is that exception, clear the error, and remember the requesting command; the completion pass then skips the requesting command and every command before it, and still completes the ones after it (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.DoCompleteCore@L614-L709). `INFERENCE`: the requester has already finished its own work when it asked upstream to stop. + +**Concurrency check on invoke** (src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.DoConcurrentCheck@L617-L700): a non-nested pipeline may start only when no pipeline is running; if the running one is the internal "pulse" pipeline (or a nested pipeline while a pulse exists), the caller waits for it and re-checks; otherwise `ConcurrentInvokeNotAllowed`. A nested pipeline must be started synchronously, from the same thread that runs its parent; otherwise `NestedPipelineInvokeAsync` / `NestedPipelineNoParentPipeline`. A child pipeline with no parent running becomes a normal pipeline. + +## 8. Observable behavior [SPEC] + +- Output of a pipeline appears incrementally: a downstream command's process block runs as soon as an upstream command writes an object (default buffer threshold 0). +- `begin` blocks of all commands run before the first `process` block; `end` blocks run in command order after input ends. +- A stop (`Ctrl+C`, `Stop()`) makes running commands see `PipelineStoppedException`; `clean` blocks still run. +- One runspace runs one top-level pipeline at a time; a second concurrent invoke fails with `ConcurrentInvokeNotAllowed` (a `RunspacePool` is the way to run several). +- `Pester` specification of pipeline behavior is in the tests of §16 (`PipelineBehaviour.Tests.ps1`, 29 `It` blocks). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +- **First error wins**: `RecordFailure` stores the first terminating error under a lock; a second different failure is logged as a health event with both stack traces and does not replace the first; the first call also sets `_stopping`. +- **Command-level errors**: pipeline-object binding errors are written as non-terminating errors with the object as target, then the object is skipped (§6 CF-LCY-001); `InputObjectNotBound` is raised when no parameter accepts the object (src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Read@L452-L581). +- **Disposal errors**: exceptions thrown while disposing a command are wrapped (`CmdletInvocationException` / `CmdletProviderInvocationException`), logged as warnings and recorded as failures (src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.DisposeCommands@L1347-L1445). +- **Exit and flow control** at the top level: `exit` records an exit code and asks the host to exit (non-nested) or sets `$LASTEXITCODE` and leaves the nested prompt (nested); other flow-control exceptions are swallowed unless the caller asked to expose them (`ExposeFlowControlExceptions`), in which case the pipeline ends `Failed` with that exception (src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.InvokeHelper@L276-L526). +- **Pipeline end state**: `Completed` on normal end or `HaltCommandException`; `Stopped` on `PipelineStoppedException`; `Failed` on `RuntimeException`, `ScriptCallDepthException` or `SecurityException` (src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.InvokeThreadProc@L551-L726). After the state is set: history entry added (unless an incomplete-parse error), output/error/input streams closed (not for child pipelines), external stream pointers restored, runspace running list updated, state events raised (asynchronous case). +- **Runspace open failure**: any exception while creating the engine sets `Broken` and rethrows; a failure while applying the initial session state also sets `Broken` (src/System.Management.Automation/engine/hostifaces/LocalConnection.cs::LocalRunspace.DoOpenHelper@L628-L721). + +## 10. Concurrency and threading + +- **Where a pipeline runs** is chosen by `PSThreadOptions` (nested pipelines always use the current thread) (src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.StartPipelineExecution@L132-L252): `Default`/`UseNewThread` start a new thread with a **10,000,000-byte stack** (`DefaultPipelineStackSize`) and the runspace apartment state; `ReuseThread` hands the work to the runspace's single long-lived worker thread (or runs inline when nested); `UseCurrentThread` runs on the caller's thread and restores the previous nested-pipeline thread and culture afterwards. The runspace default is `Default`; a pool creates its runspaces with `ReuseThread` when its own option is `Default` (src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs::RunspacePoolInternal.CreateRunspace@L1257-L1292). `ReuseThread` is rejected for STA runspaces through `IsValidThreadOptionsConfiguration`. +- **Thread-local state**: the current runspace is `Runspace.DefaultRunspace` (set for the duration of the pipeline thread and restored afterwards), and `LocalPipeline.GetExecutionContextFromTLS` reads the context from it. +- **Locks**: the runspace takes its `SyncRoot` for state changes and for the concurrency check; state events are queued under the lock and raised after releasing it so subscribers can call back. `PipelineStopper` has its own lock. `RecordFailure` has `_stopReasonLock`. +- **Session-state proxy**: while a `SessionStateProxy` call is in progress no pipeline may start and no second proxy call may start; a proxy call waits out a pulse pipeline but fails if a real pipeline runs (`NoSessionStateProxyWhenPipelineInProgress`) (src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs::RunspaceBase.DoConcurrentCheckAndMarkSessionStateProxyCallInProgress@L1168-L1223). +- **Pulse pipeline**: `Pulse()` creates and runs an internal empty pipeline (id "0") when nothing runs, so queued engine events and pending actions can be processed (`ProcessPendingActions` runs in the pipeline's `finally`). +- **Pool**: `GetRunspace` blocks on an async result; `ReleaseRunspace` returns an opened runspace to the pool stack if the pool is below its maximum, otherwise destroys it; a runspace not in `Opened` state is destroyed (src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs::RunspacePoolInternal.ReleaseRunspace@L753-L814). +- **`PowerShell` API**: `CoreInvokeHelper` gets a runspace (from the pool, blocking, or its own) and checks it is `Opened` before doing the work; failure sets `PSInvocationState.Failed`. `CoreStop` mirrors the pipeline stop table: `NotStarted` → Stopping + Stopped event; `Running` → Stopping then the worker stops; `Disconnected` → `Failed`; already-final states return a completed result (src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PowerShell.CoreStop@L5078-L5197). + +State diagrams (transitions read in code; `Broken` and the remoting states are reachable only as noted): + +```mermaid +stateDiagram-v2 + [*] --> BeforeOpen + BeforeOpen --> Opening: Open (CoreOpen) + BeforeOpen --> Closing: Close (never opened) + Opening --> Opened: engine created and initial session state bound + Opening --> Broken: engine creation or binding failed + Opened --> Closing: Close + Closing --> Closed: pipelines finished + Closed --> [*] + Broken --> [*] +``` + +```mermaid +stateDiagram-v2 + [*] --> NotStarted + NotStarted --> Running: Invoke (concurrency check passed) + NotStarted --> Stopping: Stop + Stopping --> Stopped: no pipeline started + Running --> Completed: normal end or HaltCommandException + Running --> Failed: RuntimeException or exposed flow control + Running --> Stopping: Stop + Stopping --> Stopped: PipelineStoppedException + Completed --> [*] + Failed --> [*] + Stopped --> [*] +``` + +## 11. Extension points and customization hooks + +- `PSThreadOptions`, `ApartmentState`, `InvocationSettings` (host override, `ExposeFlowControlExceptions`, `FlowImpersonationPolicy`, apartment state) per invocation. +- Runspace/pipeline/availability **events** (`StateChanged`, `AvailabilityChanged`) for observers. +- Pipeline streams can be supplied by the caller (`PSDataCollection` input/output), which is how the SDK streams results. +- `PipelineProcessor` accepts external reader/writers and *redirection pipelines*; commands may be added as `CommandInfo` objects directly (alias chains are resolved through `ReferencedCommand`; cmdlet, script/function, application only) (src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.CreateCommandProcessBase@L926-L954). +- Policy hook candidates for the new shell are listed in §18 (they are the places where this code already decides *which command*, *which thread*, *which language mode*). + +## 12. Dependencies + +Inbound: hosts (`ConsoleHost` `Executor`), SDK users, remoting servers, `Invoke-Command`/jobs, nested prompts. Outbound: `ExecutionContext` and session state (T29), `CommandDiscovery` (T32; used by `Command.CreateCommandProcessor`), parameter binders (T33), `MshCommandRuntime` (stream writing, T33), the event manager (T39), history, `PSHostUserInterface` (transcription calls), `MshLog` (lifecycle and health events), ETW activity ids. + +## 13. Performance notes + +- Per-command objects are created per invocation: a `MshCommandRuntime`, pipes, binder controller. Cmdlet instances are created through a compiled `new` expression cached per type in a concurrent dictionary, with 13 frequently used cmdlets pre-seeded (`ForEach-Object`, `Where-Object`, `Import-Module`, `Get-Module`, `Get-Help`, `Invoke-Command`, `Get-Command`, `Out-Default`, `Out-Host`, `Out-Null`, `Set-StrictMode`, `Format-Default`, `Out-LineOutput`) (src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.ConstructInstance@L666-L678). +- The synchronous push model avoids queues and threads inside a pipeline; the cost is deep call stacks (stack-depth check on every `DoExecute`, 10 MB thread stacks). +- A runspace pool exists to avoid creating a thread plus engine per request; `ReuseThread` avoids thread creation per pipeline. + +## 14. Security considerations + +Trust assumptions: whoever can call `Invoke` on a runspace can run any command the session state exposes; the language mode and session-state restrictions are properties of the runspace, not of the pipeline. Enforcement seen while reading: (a) `CommandProcessor.Prepare` switches the language mode to a script block's own mode while binding and flags the Constrained→Full transition for the binder (src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Prepare@L193-L254); (b) `ValidateCompatibleLanguageMode` refuses to dot-source a script block whose language mode differs from the caller's when the caller is restricted/constrained, unless the block can be converted to a safe `PowerShell` object, and under audit enforcement only logs the violation (src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.ValidateCompatibleLanguageMode@L199-L250); (c) pools call `EnforceSystemLockDownLanguageMode` on every runspace they create (src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs::RunspacePoolInternal.CreateRunspace@L1257-L1292); (d) impersonation flows to the pipeline thread only when requested (`FlowImpersonationPolicy`, Windows). Transcription hooks (`TranscribeCommand`, `TranscribeCommandComplete`, `TranscribePipelineComplete`) are called from pipeline execution, and a transcribing session appends `Out-Default` to a top-level pipeline that lacks one. + +### Security surface + +Scanner seeds under this subsystem's paths (unverified; no `SEC-ENF` ID is assigned here, Phase 5 does that): + + +| Category | Seed hits | +|---|---:| +| logging | 95 | +| language-mode | 42 | +| securestring | 7 | +| amsi | 2 | +| app-control | 1 | +| **total** | 147 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs` | 91 | +| `src/System.Management.Automation/engine/hostifaces/Command.cs` | 20 | +| `src/System.Management.Automation/engine/hostifaces/LocalConnection.cs` | 8 | +| `src/System.Management.Automation/engine/hostifaces/InternalHost.cs` | 6 | +| `src/System.Management.Automation/engine/hostifaces/InternalHostUserInterface.cs` | 6 | +| `src/System.Management.Automation/engine/hostifaces/PSTask.cs` | 6 | +| `src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs` | 4 | +| `src/System.Management.Automation/engine/hostifaces/Connection.cs` | 3 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: (1) the decision "which command will run" and "in which language mode" is made once per command, before binding, in one place; (2) a command cannot write to the pipeline outside its begin/process/end/clean windows; (3) a stop request reaches every active command and cannot be swallowed by user code; (4) a pool must apply the same lockdown to every runspace it creates; (5) the audit/transcription calls cover every top-level pipeline. + +## 15. Enterprise considerations + +Security first: see §14. Observability: command lifecycle events (`Started`, `Stopped`, `Terminated`) go to `MshLog` with an ETW activity id per command; per-command execution detail can be buffered and written as one event when logging is enabled; transcription hooks exist. Supportability: state events and `InvalidPipelineStateException` carry current and expected states. Testability: the engine is hard to test in isolation because `PipelineProcessor` needs a live `ExecutionContext`. Resource limits: thread stack size is fixed (10 MB); no limit on queued objects between commands; stream capacity is unbounded by default; no per-pipeline memory or time limit is enforced here. Missing: cancellation by time budget, a quota on running pipelines per host, structured per-command resource accounting. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Language/Classes/Scripting.Classes.NoRunspaceAffinity.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/Language/Operators/PipelineChainOperator.Tests.ps1` | 1 | 2 | 7 | +| `test/powershell/Language/Scripting/PipelineBehaviour.Tests.ps1` | 2 | 5 | 29 | +| `test/powershell/Language/Scripting/PipelineStoppedToken.Tests.ps1` | 1 | 0 | 2 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/RunspaceCmdlets.Tests.ps1` | 1 | 1 | 5 | +| `test/powershell/engine/Api/BasicEngine.Tests.ps1` | 3 | 2 | 9 | +| `test/powershell/engine/Api/InitialSessionState.Tests.ps1` | 2 | 2 | 8 | +| `test/powershell/engine/Api/PSCommand.Tests.ps1` | 1 | 2 | 4 | +| `test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1` | 1 | 4 | 9 | +| `test/powershell/engine/Remoting/RunspacePool.Tests.ps1` | 1 | 0 | 1 | +| **total (10 files)** | 14 | 18 | 75 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_PowerShellAPI.cs` | 1 | 0 | `PowerShellHostingScenario` | +| `test/xUnit/csharp/test_Runspace.cs` | 3 | 0 | `RunspaceTests` | + +Matching is by path keyword (`APPROX`): `PipelineBehaviour`, `PipelineStoppedToken`, `PipelineChainOperator`, `RunspaceCmdlets`, `engine/Remoting/RunspacePool`, `TaskBasedAsyncPowerShellAPI`, `NoRunspaceAffinity`, `engine/Api/PSCommand`, `engine/Api/BasicEngine`, `engine/Api/InitialSessionState`, `test_Runspace`, `test_PowerShellAPI`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: the engine pipeline is synchronous and thread-affine; concurrency exists only at the runspace level (one thread per running pipeline) and in explicit constructs (jobs, `ForEach-Object -Parallel`, pools). +- `INFERENCE`: pushing objects depth-first gives streaming output with no buffering and no scheduler, at the price of deep stacks and no parallelism between pipeline stages. +- `FACT-SRC`: begin blocks run for all commands before any process block, so a downstream `begin` can observe state produced by an upstream `begin`, but never data. +- `FACT-SRC`: state-change events are queued under the lock and raised outside it, with handler exceptions swallowed. +- `INFERENCE`: the "pulse pipeline" exists to let engine events run when the runspace is idle. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-RTPIPE-01** [REPLICATE] (area: NLD-PIP) Keep the synchronous depth-first push model with a zero default buffer threshold: it defines the observable streaming behavior of pipelines. Evidence: src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466. +- **LSN-RTPIPE-02** [REPLICATE] (area: NLD-PIP) Bind all commands, then run all begin phases in order, then process, then end in order; keep the clean phase outside the stop path. Evidence: src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147. +- **LSN-RTPIPE-03** [REPLICATE] (area: NLD-PIP) First terminating error wins; later failures are logged, not thrown. Evidence: src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.RecordFailure@L1455-L1498. +- **LSN-RTPIPE-04** [ADAPT] (area: NLD-PIP) Keep the one-pipeline-per-runspace rule but make the state machines explicit tables with a single transition function; the original spreads transitions over several classes with events queued under locks. Evidence: src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.SetPipelineState@L798-L823. +- **LSN-RTPIPE-05** [ADAPT] (area: NLD-PIP) Replace the fixed 10 MB thread stack plus per-call stack probe with a configurable depth limit that raises the same `ScriptCallDepth` error. Evidence: src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline@L21-L1143. +- **LSN-RTPIPE-06** [REDESIGN] (area: NLD-PIP) Offer a bounded, cancellable stream between host and pipeline by default (the original default is effectively unbounded) and a time/memory budget per pipeline, as an opt-in deviation. DEV: DEV-PIPE-110. Evidence: src/System.Management.Automation/utils/ObjectStream.cs::ObjectStream@L395-L1517. +- **LSN-RTPIPE-07** [REPLICATE] (area: NLD-SEC) Create the policy decision point at command-processor construction and before parameter binding: command origin, language mode and session-state visibility are all known there. Evidence: src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.CreatePipelineProcessor@L817-L919. +- **LSN-RTPIPE-08** [AVOID] (area: NLD-PIP) Do not reproduce the "pulse pipeline" (an internal empty pipeline used to run pending engine events); schedule event processing on the runspace's own loop. DEV: DEV-PIPE-111. Evidence: src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs::RunspaceBase.Pulse@L1094-L1150. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-024): capacity and thread-safety of the three `PipelineBase` object streams; behavior of `ObjectWriter.Write` under a full stream. +- `UNVERIFIED` (VB-025): `PowerShell.cs` outside `CoreInvokeHelper`, `CoreStop` and the state enums (about 6,000 lines; the API surface was taken from signatures only); `RunspacePool` public wrapper and async get/cancel paths; `PSTask`; remote branches. +- `UNVERIFIED` (VB-026): `SteppablePipeline` (uses `StartStepping`/`Step`), the `MshCommandRuntime` stream-write rules, `DlrScriptCommandProcessor` begin/process/end handling inside this lifecycle (covered in T29/T210 documents). + +Verification: all non-comment lines of `Pipe.cs`, `pipeline.cs`, `CommandProcessorBase.cs` and `CommandProcessor.cs` were read; of the Tier B files only the ranges cited here were read (see ledger proposal `T31_T39.csv`). Enum members were read from source. Test counts come from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/09-synthesis/deviation-register.md b/docs-out/09-synthesis/deviation-register.md index f0de978..f42aa11 100644 --- a/docs-out/09-synthesis/deviation-register.md +++ b/docs-out/09-synthesis/deviation-register.md @@ -22,16 +22,39 @@ All rows are `PROPOSED` (the only status this run may write). `APPROVED`/`REJECT |---|---|---|---|---|---|---|---|---| | DEV-AST-020 | PROPOSED | AST nodes carry mutable compiler caches (TupleIndex, RuntimeUsingIndex, TypeDefinitionAst.Type) and implement compile-time assignment protocols (ISupportsAssignment/IAssignableValue). | AST stays purely syntactic; slot indexes and assignment code generation live in compiler side tables keyed by node. | Immutable shareable trees; clean separation of parser and compiler; easier tooling reuse. | None observable by scripts; internal refactor only. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-AST-07 (02-language/02.3-ast.md) | | DEV-BIND-050 | PROPOSED | Member access goes through COM and ETS-adapter aware binders. | Plain reflection member lookup; no COM binders. | Smaller binder layer. | No COM objects. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-BIND-03 (02-language/BINDER-CATALOG.md) | +| DEV-BIND-130 | PROPOSED | Execution-detail logging records bound values (first 256 characters) for every parameter. | Redact bound values by parameter type and attribute (secure types, sensitive marker). | Avoids leaking secrets into logs. | Existing log consumers lose raw values. | CF-BND-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTBIND-07 (03-runtime/parameter-binding.md) | | DEV-CMD-120 | PROPOSED | Windows runtimes ship `Get-Counter`, `Get-WinEvent`, `New-WinEvent` in the default command set (assembly Microsoft.PowerShell.Commands.Diagnostics). | Do not ship OS-specific diagnostics cmdlets in the default set; offer them as optional modules. | Smaller default attack surface and a portable base command set. | Scripts calling these cmdlets need the optional module installed. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-DIAG-01 (10-command-modules/Microsoft.PowerShell.Commands.Diagnostics.md) | | DEV-CMD-121 | PROPOSED | WSMan client cmdlets and provider (assembly Microsoft.WSMan.Management) are part of the Windows command set. | Omit WSMan commands; remoting transport becomes pluggable behind one session abstraction. | No dependency on a Windows-only protocol stack in the base product. | WSMan-based automation breaks; an optional transport module could restore it. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-WSMAN-01 (10-command-modules/Microsoft.WSMan.Management.md) | | DEV-CMD-122 | PROPOSED | CIM/WMI cmdlets (assembly Microsoft.Management.Infrastructure.CimCmdlets) ship on Windows. | Provide CIM/WMI access only as an optional adapter module. | Keeps the base command set free of Windows management stacks. | Windows management scripts need the optional module. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CIM-01 (10-command-modules/Microsoft.Management.Infrastructure.CimCmdlets.md) | +| DEV-CMDLET-135 | PROPOSED | Cmdlets can require a transaction (-UseTransaction, SupportsTransactions) via an engine protection scope. | Omit transaction support. | Removes a Windows-only feature and engine scope handling. | Cmdlets and scripts using transactions are unsupported. | CF-LCY-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTCMD-06 (03-runtime/cmdlet-infrastructure.md) | | DEV-CTX-081 | PROPOSED | ExecutionContext carries language mode and invalidates binder caches on change. | No language modes. | No cache invalidation logic. | No restricted modes. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CTX-04 (03-runtime/execution-context-and-session-state.md) | +| DEV-DBG-195 | PROPOSED | Debugger enable/attach is not a distinct audited privilege. | Policy-controlled, audited debugger attach; prompt runs at the lowest trust level on the stack. | Controls a code-execution channel. | Debug sessions need explicit permission. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTDBG-04 (03-runtime/debugger.md) | +| DEV-DBG-196 | PROPOSED | Nested debugging of jobs and remote runspaces exists. | Omit in the first version. | Smaller debugger. | No job or remote debugging initially. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTDBG-05 (03-runtime/debugger.md) | | DEV-DESIGN-110 | PROPOSED | PowerShell compiles to expression trees and uses Reflection.Emit for classes. | MicroteamShell starts with a tree-walking evaluator and does not adopt Reflection.Emit or Expression.Compile as the core engine. | AOT-friendly, debuggable, small. | Lower peak throughput; classes (if added) are interpreter objects. | CF-CMP-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-DESIGN-03 (02-language/INTERPRETER-DESIGN-SPACE.md) | +| DEV-DISC-120 | PROPOSED | Unknown command names trigger module auto-discovery by analyzing every module file on PSModulePath. | Resolve through an explicit signed command index refreshed on module install/update; keep module-qualified auto-load. | Faster misses, no file scans on typos, auditable resolution. | Modules not indexed are not auto-loaded until indexed. | CF-DSC-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTDISC-05 (03-runtime/command-discovery-and-invocation.md) | +| DEV-DISC-121 | PROPOSED | Native argument passing has three styles (Legacy, Standard, Windows) with a name-based list of legacy programs. | List-based passing only, with a per-command opt-in for raw command lines. | Removes the quoting ambiguity and the legacy program list. | Scripts relying on Legacy re-quoting need the opt-in. | CF-DSC-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTDISC-06 (03-runtime/command-discovery-and-invocation.md) | | DEV-ENG-040 | PROPOSED | Compilation of a top-level script block runs an AMSI scan, admin-block check and suspicious-content hashing. | Compilation is policy-free; an optional policy hook runs before execution. | Smaller engine core; platform-independent. | No AMSI integration or forced logging on suspicious content. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-ENG-07 (02-language/02.5-compiler-and-engine.md) | | DEV-EQV-061 | PROPOSED | Interpreter keeps typed arithmetic/comparison instruction families inherited from the DLR. | Do not port them; numeric semantics come from the evaluator's conversion rules. | Removes dead and defective code. | None observable (instructions unreachable from PowerShell IR). | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-EQV-03 (02-language/EQUIVALENCE-SPEC.md) | +| DEV-ERR-180 | PROPOSED | Errors are only on the user error stream and honour -ErrorAction/$ErrorActionPreference, including suppression. | A separate audit stream for policy decisions and denials that user preferences cannot suppress. | Audit integrity. | Additional stream to consume. | CF-ERR-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTERR-07 (03-runtime/errors-and-diagnostics.md) | +| DEV-ETS-150 | PROPOSED | Adapters exist for WMI, DirectoryEntry, DataRow/DataRowView, COM, CIM and XML nodes. | Ship .NET, XML, PSObject, member-set and the third-party adapter bridge only. | Smaller surface, fewer Windows-only dependencies. | Scripts using those object kinds lose adapted members. | CF-ETS-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTETS-06 (03-runtime/extended-type-system.md) | +| DEV-ETS-151 | PROPOSED | Built-in type data is compiled into C# from types.ps1xml. | Compile at build time and also publish a versioned data file. | One source for tooling and a clean-room implementation. | None observable. | CF-ETS-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTETS-07 (03-runtime/extended-type-system.md) | +| DEV-EVT-198 | PROPOSED | Received-event and action queues are unbounded. | Bounded queues with policy-defined drop rules. | Resource safety. | Events may be dropped under load. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTEVT-04 (03-runtime/eventing.md) | +| DEV-EVT-199 | PROPOSED | Forwarded events and WMI/CIM event sources are supported. | Omit in the first version. | Smaller eventing surface. | No event forwarding initially. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTEVT-05 (03-runtime/eventing.md) | | DEV-FEAT-091 | PROPOSED | workflow/parallel/sequence/inlinescript parse and then fail in semantic checks; from/define/var are reserved. | Reject at parse time. | No dead syntax. | Different error text. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-FEAT-03 (02-language/02.7-language-features.md) | +| DEV-FMT-160 | PROPOSED | Output is text-only through a LineOutput sink. | Add an optional structured (typed record) sink next to the text sink. | Machine consumers need not parse text. | Additional sink contract to maintain. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTFMT-05 (03-runtime/formatting-and-output.md) | +| DEV-FMT-161 | PROPOSED | Format data is loaded from an XML schema with a large loader family. | Built-in views as compiled data and a small JSON/YAML definition for extensions. | Smaller loader, simpler validation. | Existing format.ps1xml files need conversion. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTFMT-06 (03-runtime/formatting-and-output.md) | | DEV-GRM-010 | PROPOSED | The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing. | Parsing is pure: directives are recorded in the AST and resolved in a later phase. | Deterministic, side-effect-free parse; safe for tooling and untrusted text. | Errors for a missing assembly appear at a later phase than in PowerShell. | CF-GRM-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PARSER-07 (02-language/02.2-parser.md); LSN-QRK-03 (02-language/SEMANTIC-QUIRKS.md) | +| DEV-HELP-165 | PROPOSED | Updatable help downloads unsigned packages from a module-provided URI. | Signed help packages with a catalog and offline mode, verified before install. | Integrity of installed help. | Help authors must sign packages. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTHELP-04 (03-runtime/help-system.md) | +| DEV-HELP-166 | PROPOSED | Help is authored in MAML XML plus a DSC resource provider. | Lighter structured format and comment-based help only. | Simpler authoring and parsing. | Existing MAML help needs conversion. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTHELP-05 (03-runtime/help-system.md) | +| DEV-JOB-190 | PROPOSED | Background jobs start a separate pwsh process and parallel tasks use runspaces created from the session; policy inheritance is not enforced in the code read. | Child processes and parallel runspaces inherit policy, language mode and visibility as a hard rule, with per-job quotas. | Prevents escape from a restricted session through jobs. | Jobs cannot be more privileged than the parent. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTJOB-04 (03-runtime/jobs-and-concurrency.md) | +| DEV-JOB-191 | PROPOSED | ThrottlingJob, ContainerParentJob and a job source-adapter registry exist. | One local job type plus the parallel pool. | Smaller job model. | Third-party job sources unsupported initially. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTJOB-05 (03-runtime/jobs-and-concurrency.md) | | DEV-LEX-001 | PROPOSED | Tokenizer detects signature-block lookalikes and processes #requires comment lines at parse time. | Omit both from the tokenizer; if wanted, handle pragmas in a dedicated post-tokenize pass. | Keeps the tokenizer a pure text-to-token function; removes security-sensitive fuzzy matching from the hot path. | Scripts using #requires get no version/module check; signed-script trailer detection unavailable. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOK-07 (02-language/02.1-tokenizer.md) | +| DEV-MOD-170 | PROPOSED | PSModulePath entries have equal trust and auto-load picks the first module exporting a name. | Module search path entries carry trust levels; auto-load only from trusted entries unless policy allows. | Prevents shadowing through writable low-trust folders. | Auto-load from user-writable folders needs explicit policy. | CF-MOD-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTMOD-05 (03-runtime/modules.md) | +| DEV-MOD-171 | PROPOSED | Windows PowerShell compatibility imports, .cdxml CIM modules and DSC resource exports are supported. | Omit in the first version. | Smaller scope; removes Windows-only machinery. | Those modules cannot be imported. | CF-MOD-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTMOD-07 (03-runtime/modules.md) | +| DEV-PIPE-110 | PROPOSED | Host-facing object streams are effectively unbounded (capacity Int32.MaxValue) and a pipeline has no time or memory budget. | Bounded, cancellable host streams by default plus an optional per-pipeline time/memory budget. | Predictable resource use in enterprise hosts. | Behavior differs for producers that exceed the bound; hosts must handle blocking or rejection. | CF-LCY-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPIPE-06 (03-runtime/runspace-and-pipeline.md) | +| DEV-PIPE-111 | PROPOSED | An internal empty "pulse" pipeline is created to run pending engine events when the runspace is idle. | Schedule event processing on the runspace loop without creating a pipeline. | Removes a special pipeline and its state-machine corner cases. | Code that observed the pulse pipeline (id 0, history) would not see it. | CF-LCY-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPIPE-08 (03-runtime/runspace-and-pipeline.md) | +| DEV-PROV-140 | PROPOSED | Providers receive a mutable Context set by the engine and have no policy mediation layer. | A mandatory policy mediation layer between engine and providers and an explicit per-call context object. | Root confinement, rights and audit enforced outside provider code. | Provider authors must target a new context type. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPROV-03 (03-runtime/provider-model.md) | +| DEV-PROV-141 | PROPOSED | File-system drives are auto-mounted on lookup miss. | Mount drives explicitly or from configuration. | Predictable drive table; no hidden system probing. | Scripts using unmounted drive letters need explicit mounts. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPROV-06 (03-runtime/provider-model.md) | | DEV-RTOPS-051 | PROPOSED | Classes are emitted with Reflection.Emit and method bodies wrapped by ScriptBlockMemberMethodWrapper. | Classes are interpreter objects, not CLR types. | No Reflection.Emit dependency. | Classes are not usable from other .NET code. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTOPS-05 (02-language/RUNTIME-OPERATIONS.md) | | DEV-SB-090 | PROPOSED | ScriptBlock can become a PowerShell object and a steppable pipeline. | Omitted initially. | Less surface. | No GetPowerShell/GetSteppablePipeline. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SB-05 (02-language/SCRIPT-BLOCKS.md) | | DEV-SCP-080 | PROPOSED | Values assigned to global variables are marked untrusted in Constrained Language Mode. | No trust marking. | Simpler assignment path. | No Constrained Language Mode. | CF-SCP-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SCOPE-07 (02-language/SCOPE-AND-VARIABLES.md) | diff --git a/docs-out/CONTRADICTIONS.md b/docs-out/CONTRADICTIONS.md index 90619ec..40e15ec 100644 --- a/docs-out/CONTRADICTIONS.md +++ b/docs-out/CONTRADICTIONS.md @@ -17,3 +17,4 @@ Rows are never resolved by silent rewording. Status: `OPEN | RESOLVED`. Final st | CON-011 | `Compiler.BuildHashtable` creates hashtables with `OrdinalIgnoreCase` keys (`src/System.Management.Automation/engine/parser/Compiler.cs@L6533-L6546`) | `HashtableOps.Add` builds the result of `+` with `CurrentCultureIgnoreCase` (`src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs@L1431-L1456`) | E1 code vs E1 code | OPEN | observation recorded in RUNTIME-OPERATIONS §6.3; behavior differs only for culture-sensitive case folding (e.g. Turkish i); needs `pwsh` under such a culture. | docs-out/02-language/RUNTIME-OPERATIONS.md | | CON-012 | `StringOps.Multiply` comment: negative count "should be a runtime error" (`src/System.Management.Automation/engine/runtime/Operations/StringOps.cs@L27-L35`) | code throws `ArgumentOutOfRangeException` | E1 comment vs E1 code | RESOLVED | code is authoritative; documented in RUNTIME-OPERATIONS §6.3. | docs-out/02-language/RUNTIME-OPERATIONS.md | | CON-013 | earlier study documents (02.5 §4, TIERING-POLICY §2, SCRIPT-BLOCKS §4) said `ScriptBlock.InvokeWithPipeImpl` is the one execution entry for functions, scripts, `&`, `.` and `Invoke` | reading `Command.CreateCommandProcessor` and `DlrScriptCommandProcessor` shows command invocation of script text, functions and scripts runs through command processors; `InvokeWithPipeImpl` serves the `ScriptBlock.Invoke*` API (`src/System.Management.Automation/engine/hostifaces/Command.cs@L450-L540`, `src/System.Management.Automation/engine/ScriptCommandProcessor.cs@L232-L625`) | document vs E1 code | RESOLVED | documents corrected in the same commit; both paths apply the same variant rule (CF-CMP-003) | docs-out/02-language/02.5-compiler-and-engine.md, TIERING-POLICY.md, SCRIPT-BLOCKS.md, LIFECYCLE-script-execution.md | +| CON-014 | study prompt hint: command types include `Workflow` | `CommandTypes` has Alias, Function, Filter, Cmdlet, ExternalScript, Application, Script, Configuration, All (`src/System.Management.Automation/engine/CommandInfo.cs::CommandTypes@L23-L76`) | prompt hint vs E1 | RESOLVED | source wins; documented in command-discovery-and-invocation §4. | docs-out/03-runtime/command-discovery-and-invocation.md | diff --git a/docs-out/DECISIONS.md b/docs-out/DECISIONS.md index 1ef001d..5f9c386 100644 --- a/docs-out/DECISIONS.md +++ b/docs-out/DECISIONS.md @@ -22,3 +22,7 @@ | D-018 | T24, T25 and T27 were authored in one batch (inline, no subagents). Drafts use `[[file::Type.Member]]` tokens expanded by `tools/expand_refs.py` from `data/types.json` and `data/members-tierA.json`, so no reference range is typed by hand; unresolved tokens fail the expansion. The T27 documents own CF-CMP-001/002/003; `AlwaysCompile` is documented as unused after `rg` over `engine/`. | Rule 1 (evidence) and rule 9 (generated catalogs). | | D-019 | T26, T28, T29, T210 and T211 were authored inline (no subagents) in one batch on a branch restarted from the merged `main` (`git checkout -B … origin/main`). No reading notes existed for these areas, so each document states exactly which windows were read in its Verification line and §19; files only partly read are recorded with partial ranges in the ledger (`.work/ledger-proposals/T26_T211.csv`), not as fully read. Coverage for `Binders.cs` (8,015 lines), `LanguagePrimitives.cs` (5,888) and `MiscOps.cs` (3,765) is therefore partial; the gaps are backlog items VB-017..VB-023. | Rule 7 (read windows, no false coverage claims). | | D-020 | The coordinator documents (pipeline map, stage contracts, call graphs, lifecycle, spec draft, quirks, design space, golden-samples index, oracle protocol, test corpus) were authored inline on a branch restarted from the merged `main`. The lifecycle trace read the host-to-engine path (`Executor`, `LocalPipeline`, `Command.CreateCommandProcessor`, `AutomationEngine.ParseScriptBlock`, `DlrScriptCommandProcessor`), which corrected an earlier statement about the execution entry (CON-013). `data/interpreter-scores.json` is a hand-authored dataset (scores and justifications with labels); totals, gates, recommendations and sensitivity are computed by `tools/gen_coord.py`. Golden samples and oracle remain empty/waived (VB-001). | Rule 1 (evidence), rule 9 (computed catalogs), prompt 2.10 scoring rules. | +| D-021 | Phase 3 (T31-T39) documents were authored inline (no subagents), reading source through comment-stripped views (`grep -nvE` on blank/comment lines keeps line numbers). Tier A files named in the plan (`pipeline.cs`, `Pipe.cs`, `CommandProcessorBase.cs`, `CommandProcessor.cs`, `CommandSearcher.cs`, parts of `CommandDiscovery.cs`, `NativeCommandProcessor.cs`, `ParameterBinderController.cs`, `CmdletParameterBinderController.cs`, `ParameterBinderBase.cs`) were read for the algorithm ranges recorded in `.work/ledger-proposals/T31_T39.csv`; Tier B subsystems (providers, ETS, formatting, help, modules, errors, jobs, debugger, eventing) were read at contract and key-algorithm level only, each document lists its read ranges and VB-024..VB-057 list the rest. | Context budget; honesty about reading depth (same approach as D-019). | +| D-022 | `docs-out/tools/gen_T3.py` with `docs-out/tools/annotations/T3-docs.json` renders the Pester/xUnit test tables (``) and the per-subsystem security-seed tables (``) of all 03-runtime documents from `data/tests.json` and `data/enforcement-points.json`. Seed hits are scanner output, unverified, and carry no `SEC-ENF` ID; Phase 5 assigns IDs. No `SEC-ENF-*` ID is invented in Phase 3. | Rule 9 (no hand-typed catalogs) and rule 1 (evidence only). | +| D-023 | CF-LCY-001 is closed (statement fully read); CF-DSC-001, CF-BND-001, CF-ETS-001, CF-MOD-001, CF-ERR-001 stay open because parts of their algorithms are listed in VB-027, VB-030, VB-038, VB-046, VB-049. CF-ERR-001 references the statement-level protocol documented in RUNTIME-OPERATIONS §6.1 instead of restating it. | CF closure requires a fully read statement (brief rule 5). | +| D-024 | The study prompt lists `Workflow` among command types; `CommandTypes` in v7.6.9 has no such member (CON-014). The native process start point (prepare phase) and the 256-entry `$Error` cap are recorded as claims (`T3.jsonl`). | Source wins over prompt hints. | diff --git a/docs-out/PROGRESS.md b/docs-out/PROGRESS.md index 9cce03f..c0a76d4 100644 --- a/docs-out/PROGRESS.md +++ b/docs-out/PROGRESS.md @@ -9,7 +9,7 @@ Run mode: STATIC-ONLY (`ALLOW_STATIC_ONLY=true`). Branch `claude/repo-review-exp | 0 Recon, pinning, tooling | done | datasets in `data/`, tools in `tools/`, registers, glossary, census, dependency graph, source-tree map, 297 folder cards + 40 project cards (generated; annotation overlay pending) | | 1 Repository map | done | layered-architecture, build-system, ci-cd-and-packaging, coding-standards (committed 4a6efaf) | | 2 Language pipeline | in progress | wave 1: T21 DONE (draft); T22 DONE (draft); T23 DONE (draft); T24 T25 T27 DONE (draft); T26 T28 T29 T210 T211 DONE (draft, partial reading depth: see D-019 and VB-017..VB-023); the coordinator documents of 02-language are DONE (draft): pipeline map, stage contracts, call graphs, lifecycle, spec draft, 32 quirks, design space, golden-samples index (empty), oracle protocol (waived), test corpus; Phase 2 documents are complete except the reading-depth gaps VB-017..VB-023 | -| 3 Runtime | todo | | +| 3 Runtime | in progress | T31..T39 DONE (draft, tier B subsystems at contract depth): runspace-and-pipeline, command-discovery-and-invocation, parameter-binding, cmdlet-infrastructure, provider-model, extended-type-system, formatting-and-output, help-system, modules, errors-and-diagnostics, jobs-and-concurrency, debugger, eventing; execution-context-and-session-state done in Phase 2 (T29); reading gaps VB-024..VB-057; SEC-ENF IDs not yet assigned (Phase 5) | | 4 Hosting/SDK/remoting/platform/command modules | todo | | | 5 Security | todo | | | 6 Engineering | todo | | @@ -37,4 +37,16 @@ T21 tokenizer | T22 parser/grammar/errors | T23 AST | T24 semantic | T25 compile | COORD | `02-language/00-pipeline-map.md`, `STAGE-CONTRACTS.md`, `CALL-GRAPHS.md` (`tools/gen_coord.py`), `LIFECYCLE-script-execution.md`, `LANGUAGE-SPEC-DRAFT.md`, `SEMANTIC-QUIRKS.md`, `INTERPRETER-DESIGN-SPACE.md` (scores in `data/interpreter-scores.json`), `GOLDEN-SAMPLES.md`, `ORACLE-PROTOCOL.md`, `REFERENCE-TEST-CORPUS.md`, `09-synthesis/new-language-design-canvas.md`, `README.md` | draft; DEV-DESIGN-110; entry-point correction recorded (CON-013) | -Next step: Phase 3 runtime (T31 runspace/pipeline, T32 command discovery, T33 parameter binding and cmdlet infrastructure, T34-T39) and the deeper reads listed in VB-017..VB-023. Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. +| T31 | `03-runtime/runspace-and-pipeline.md` (CF-LCY-001) | draft; DEV-PIPE-110/111; VB-024..026; checks pass | +| T32 | `03-runtime/command-discovery-and-invocation.md` (CF-DSC-001) | draft; DEV-DISC-120/121; CON-014; VB-027..029; checks pass | +| T33 | `03-runtime/parameter-binding.md` (CF-BND-001), `03-runtime/cmdlet-infrastructure.md` | draft; DEV-BIND-130, DEV-CMDLET-135; VB-030..034; checks pass | +| T34 | `03-runtime/provider-model.md` | draft (tier B); DEV-PROV-140/141; VB-035..037 | +| T35 | `03-runtime/extended-type-system.md` (CF-ETS-001) | draft (tier B); DEV-ETS-150/151; VB-038/039 | +| T36 | `03-runtime/formatting-and-output.md`, `03-runtime/help-system.md` | draft (tier B); DEV-FMT-160/161, DEV-HELP-165/166; VB-040..044 | +| T37 | `03-runtime/modules.md` (CF-MOD-001) | draft (tier B); DEV-MOD-170/171; VB-045..047 | +| T38 | `03-runtime/errors-and-diagnostics.md` (CF-ERR-001) | draft (tier B); DEV-ERR-180; VB-048..050 | +| T39 | `03-runtime/jobs-and-concurrency.md`, `03-runtime/debugger.md`, `03-runtime/eventing.md` | draft (tier B); DEV-JOB-190/191, DEV-DBG-195/196, DEV-EVT-198/199; VB-051..057 | + +Generated parts of 03-runtime (test tables, security-seed tables): `docs-out/tools/gen_T3.py` + `docs-out/tools/annotations/T3-docs.json` (D-022). + +Next step: Phase 4 (hosting, SDK, remoting, platform, command-module families), Phase 5 security (assign `SEC-ENF` IDs to verified seed hits), and the deeper reads in VB-017..VB-023 and VB-024..VB-057. Open: golden samples and oracle remain waived (VB-001). diff --git a/docs-out/README.md b/docs-out/README.md index 5e47fbb..7760c28 100644 --- a/docs-out/README.md +++ b/docs-out/README.md @@ -11,6 +11,12 @@ Pin: version `v7.6.9`, commit `523031e` (`.work/pin.json`). STATIC-ONLY run: not 5. Synthesis: [LANGUAGE-SPEC-DRAFT.md](02-language/LANGUAGE-SPEC-DRAFT.md), [SEMANTIC-QUIRKS.md](02-language/SEMANTIC-QUIRKS.md), [INTERPRETER-DESIGN-SPACE.md](02-language/INTERPRETER-DESIGN-SPACE.md), [02.7-language-features.md](02-language/02.7-language-features.md), [HOWTO-extend-language.md](02-language/HOWTO-extend-language.md), [02.8-language-tooling.md](02-language/02.8-language-tooling.md), [new-language-design-canvas.md](09-synthesis/new-language-design-canvas.md) 6. Evidence and verification: [REFERENCE-TEST-CORPUS.md](02-language/REFERENCE-TEST-CORPUS.md), [GOLDEN-SAMPLES.md](02-language/GOLDEN-SAMPLES.md) (empty), [ORACLE-PROTOCOL.md](02-language/ORACLE-PROTOCOL.md) (waived) +## Runtime (`03-runtime/`) + +1. Execution: [runspace-and-pipeline.md](03-runtime/runspace-and-pipeline.md) (CF-LCY-001) → [command-discovery-and-invocation.md](03-runtime/command-discovery-and-invocation.md) (CF-DSC-001) → [parameter-binding.md](03-runtime/parameter-binding.md) (CF-BND-001) → [cmdlet-infrastructure.md](03-runtime/cmdlet-infrastructure.md) → [errors-and-diagnostics.md](03-runtime/errors-and-diagnostics.md) (CF-ERR-001) +2. State and types: [execution-context-and-session-state.md](03-runtime/execution-context-and-session-state.md), [extended-type-system.md](03-runtime/extended-type-system.md) (CF-ETS-001), [modules.md](03-runtime/modules.md) (CF-MOD-001), [provider-model.md](03-runtime/provider-model.md) +3. Output and tooling: [formatting-and-output.md](03-runtime/formatting-and-output.md), [help-system.md](03-runtime/help-system.md), [jobs-and-concurrency.md](03-runtime/jobs-and-concurrency.md), [debugger.md](03-runtime/debugger.md), [eventing.md](03-runtime/eventing.md) + ## Repository overview (`00-overview/`, `01-repository/`, `10-command-modules/`) [repo-census](00-overview/repo-census.md), [source-tree-map](00-overview/source-tree-map.md), [project-dependency-graph](00-overview/project-dependency-graph.md), [external-dependencies](00-overview/external-dependencies.md), [layered-architecture](00-overview/layered-architecture.md), [build-system](01-repository/build-system.md), [ci-cd-and-packaging](01-repository/ci-cd-and-packaging.md), [coding-standards](01-repository/coding-standards.md), [command module families](10-command-modules/_index.md). diff --git a/docs-out/VERIFICATION-BACKLOG.md b/docs-out/VERIFICATION-BACKLOG.md index afeed45..998056b 100644 --- a/docs-out/VERIFICATION-BACKLOG.md +++ b/docs-out/VERIFICATION-BACKLOG.md @@ -27,3 +27,37 @@ Every `UNVERIFIED` claim and every `BLOCKED` item. Under STATIC-ONLY (owner deci | VB-021 | OPEN | LANG-RTOPS-001 §19 | `PipelineOps.InvokePipeline`, redirections, `.Where()`/`.ForEach()`/member enumeration, `TypeOps`, `SwitchOps`, `ClassOps`, `PipelineOps.Splat` not read. | Read `MiscOps.cs` L29-L890, L2199-L2590, L2623-L3420. | Runtime helper detail. | | VB-022 | OPEN | LANG-SCRIPTBLOCK-001 §19 | `ScriptBlockToPowerShellConverter` rules and `$using` rewriting not read. | Read `ScriptBlockToPowerShell.cs` (full). | Remoting/jobs support. | | VB-023 | OPEN | LANG-TOOLING-001 §19 | Completion analysis/completers (Tier C) not read. | Read `CompletionAnalysis.cs`, `CompletionCompleters.cs` if completion is wanted. | Tooling. | +| VB-024 | OPEN | RT-PIPELINE-001 §19 | Capacity and thread-safety of the three `PipelineBase` object streams; behavior of `ObjectWriter.Write` on a full stream. | Read `PipelineBase` stream construction and `ObjectStream` writer paths (`utils/ObjectStream.cs` L395-L1517). | Back-pressure claims for host streams. | +| VB-025 | OPEN | RT-PIPELINE-001 §19 | `PowerShell.cs` outside `CoreInvokeHelper`/`CoreStop`/enums (about 6,000 lines), `RunspacePool` wrapper and async get/cancel paths, `PSTask`, remote branches not read. | Read `PowerShell.cs`, `RunspacePool.cs`, `RunspacePoolInternal.cs` in windows. | SDK surface and pool semantics. | +| VB-026 | OPEN | RT-PIPELINE-001 §19 | `SteppablePipeline` (`StartStepping`/`Step`), `MshCommandRuntime` stream-write rules beyond those cited, `DlrScriptCommandProcessor` begin/process/end inside the lifecycle. | Read `ScriptCommandProcessor.cs` L232-L661 and the SteppablePipeline API. | Lifecycle for script commands. | +| VB-027 | OPEN | RT-DISCOVERY-001 §19 | `CommandPathSearch` internals (enumeration order, case rules, caching), `CommandInfo` subclasses, `GetCommandCommand` use of search options. | Read `CommandPathSearch.cs` (596 lines) and `GetCommandCommand.cs`. | Resolution detail inside a PATH directory. | +| VB-028 | OPEN | RT-DISCOVERY-001 §19 | `NativeCommandProcessor`: output queue/decoding (`InitOutputQueue`, `ProcessOutputRecord`), `KillProcess*`, byte-pipe classes, `MinishellParameterBinderController`, package-manager PATH update. | Read the unread ranges of `NativeCommandProcessor.cs` and `BytePipe.cs`, `AsyncByteStreamTransfer.cs`. | Native stream and encoding behavior. | +| VB-029 | OPEN | RT-DISCOVERY-001 §19 | `AuthorizationManager.ShouldRunInternal` algorithm (execution policy, signature rules) not read. | Read `SecurityManagerBase.cs` and the authorization manager (Phase 5). | Execution-policy semantics. | +| VB-030 | OPEN | RT-BIND-001 §19 | `GetMissingMandatoryParameters` lines 2000-2393, `NewParameterSetPromptingData`, prompt UI (`PromptForMissingMandatoryParameters`, `BuildLabel`), `CoerceTypeAsNeeded`/`EncodeCollection` (`ParameterBinderBase` L984-L1900). | Read the listed ranges. | Mandatory handling with several sets; collection coercion rules. | +| VB-031 | OPEN | RT-BIND-001 §19 | `GetDefaultParameterValuePairs` key-matching rules, `RestoreDefaultParameterValues`, `InvokeAndBindDelayBindScriptBlock` details. | Read `CmdletParameterBinderController.cs` L591-L846, L3660-L3769, L4176-L4269. | `$PSDefaultParameterValues` semantics. | +| VB-032 | OPEN | RT-BIND-001 §19 | `InternalParameterMetadata` construction from attributes and script blocks, `ScriptParameterBinder`, `ReflectionParameterBinder`, common-parameter binders. | Read `TypeMetadata.cs` L1117-L1566 and the binder classes. | Metadata limits and error ids. | +| VB-033 | OPEN | RT-CMDLET-001 §19 | Rest of `MshCommandRuntime` (verbose/debug/information/progress writers, `ManageException`, `InquireHelper`, variable setup), `PSCmdlet` helpers, `PSScriptCmdlet`, `DefaultCommandRuntime`. | Read `MshCommandRuntime.cs` in windows and `CompiledScriptBlock.cs` L2179-L2552. | Stream and consent details. | +| VB-034 | OPEN | RT-CMDLET-001 §19 | Flow of `CmdletAttribute` and `CmdletInfo` metadata into `CommandMetadata`. | Read `CommandMetadata.cs` and `CmdletInfo.cs`. | Cmdlet declaration semantics. | +| VB-035 | OPEN | RT-PROVIDER-001 §19 | `LocationGlobber` globbing internals, `GetProviderPath` variants, home-path rules, `CmdletProviderContext` members, `ProviderIntrinsics` facade, `SessionStateContainer`/`SessionStateItem` routing. | Read `LocationGlobber.cs` windows and `CoreCommandContext.cs`. | Path resolution detail. | +| VB-036 | OPEN | RT-PROVIDER-001 §19 | `FileSystemProvider` (9,600 lines), `RegistryProvider`, `CertificateProvider`, `WSManConfigProvider`; symbolic links and content encodings. | Read file-system provider windows (Tier B). | Built-in provider behavior. | +| VB-037 | OPEN | RT-PROVIDER-001 §19 | Decoration of provider items with `PSPath`, `PSParentPath`, `PSChildName`, `PSDrive`, `PSProvider` believed from type data, not read. | Read `CmdletProviderContext.WriteObject` and the type data. | Item note properties. | +| VB-038 | OPEN | RT-ETS-001 §19 | `DotNetAdapter` member discovery (overload tables, caches, hidden members, case collisions), type-name hierarchy order, `PSMethod` invocation, unread `PSObject` members, `TypeTable.Update` and types.ps1xml loading, enumeration order across sources. | Read `CoreAdapter.cs` L2616-L4655, `TypeTable.cs` Update paths. | Member view detail. | +| VB-039 | OPEN | RT-ETS-001 §19 | `PSMemberSet`/`PSPropertySet` semantics beyond lookup; `PSEvent`; `PSDynamicMember`. | Read `MshMemberInfo.cs` L3039-L3700. | Member set behavior. | +| VB-040 | OPEN | RT-FORMAT-001 §19 | `TypeMatch.PerfectMatch`/`BestMatch` ranking, complex views, `groupBy`, expression evaluation in views, `PropertyCountForTable` default. | Read `typeDataQuery.cs` L63-L242 and `displayDescriptionData*.cs`. | View selection ties. | +| VB-041 | OPEN | RT-FORMAT-001 §19 | `TableWriter`, `ListWriter`, `ComplexWriter`, `ColumnWidthManager`, `ConsoleLineOutput`, `FormattedObjectsCache` internals, `PSStyle`/`StringDecorated`. | Read the writer and cache files. | Layout rules. | +| VB-042 | OPEN | RT-FORMAT-001 §19 | XML loader and schema validation, `FormatTable` API, built-in format data contents. | Read `typeDataXmlLoader*.cs`. | Format data model. | +| VB-043 | OPEN | RT-HELP-001 §19 | `DoGetHelp`/`SearchHelp` branch rules, comment-block placement, `MUIFileSearcher` fallback, `Get-Help` parameters, `ProviderHelpProvider`, about topics, `DefaultCommandHelpObjectBuilder`. | Read `HelpSystem.cs` L260-L350, L458-L560 and the provider files. | Help lookup detail. | +| VB-044 | OPEN | RT-HELP-001 §19 | `UpdatableHelpSystem` download/install/verification, `CabinetNativeApi`, `SaveHelpCommand`. | Read `UpdatableHelpSystem.cs` and `UpdateHelpCommand.cs`. | Updatable help integrity. | +| VB-045 | OPEN | RT-MODULES-001 §19 | `SetModulePath` and the default `PSModulePath` composition, `ModuleUtils.GetModuleVersionSubfolders` ordering, `ModuleSpecification` matching rules. | Read `ModuleIntrinsics.cs` L1295-L1396, `ModuleUtils.cs`, `ModuleSpecification.cs`. | Module path and version choice. | +| VB-046 | OPEN | RT-MODULES-001 §19 | `LoadModuleManifest` (L1528-L3580), `LoadModule` (L5627-L6120), `RemoveModule` (L5022-L5270), required-module resolution (L3822-L4210), binary module loading, `PSModuleInfo` members, `Import-Module` parameters. | Read the listed ranges. | Load and removal algorithm. | +| VB-047 | OPEN | RT-MODULES-001 §19 | Where language-mode and application-control rules are applied to module loading and `RequiredAssemblies`. | Search module loader for `SystemPolicy`/language-mode checks (Phase 5). | Module trust. | +| VB-048 | OPEN | RT-ERRORS-001 §19 | `ErrorRecord` internals (fields, `ToString`, serialization, `ScriptStackTrace`), `ErrorDetails`, error-view rendering and format data, `Get-Error`. | Read `ErrorPackage.cs` L530-L1700 and the error view format data. | Error display. | +| VB-049 | OPEN | RT-ERRORS-001 §19 | Exact semantics of `ThrowTerminatingError` vs `throw` inside traps for each case; `try`/`finally` on `PipelineStoppedException`; `RethrowException`; `HaltCommandException` consumers. | Read `MiscOps.cs` ExceptionHandlingOps (L1699-L1908) and `Compiler.cs` try lowering. | Handler edge cases. | +| VB-050 | OPEN | RT-ERRORS-001 §19 | `ConsoleHost` transcript cmdlets, the policy-driven transcription path (`PSConfiguration`), `MshLog` health-event mapping. | Read `StartTranscriptCmdlet.cs`, `ConsoleHostTranscript.cs`, `PSConfiguration.cs` transcription part. | Transcript behavior. | +| VB-051 | OPEN | RT-JOBS-001 §19 | `PSRemotingJob`/`PSRemotingChildJob` internals, `ThrottlingJob`, `Job2` async methods, `JobManager` queries, `Start-Job` parameters and lockdown inheritance, `PSTaskJob`/`PSTaskChildJob`, `ForEachObjectCommand` outside the parallel setup. | Read the listed files (Tier B). | Job semantics and policy inheritance. | +| VB-052 | OPEN | RT-JOBS-001 §19 | `ThreadJob` module, scheduled jobs and workflow jobs are outside this engine and not analysed. | Study the module repositories if thread jobs are wanted. | Thread job behavior. | +| VB-053 | OPEN | RT-DEBUGGER-001 §19 | `ScriptDebugger` stop/resume machinery (`OnDebuggerStop`, `ResumeExecution`, locking, `ProcessCommand`), remote and nested debuggers. | Read `debugger.cs` L1700-L4218 and L4227-L5026. | Debugger protocol. | +| VB-054 | OPEN | RT-DEBUGGER-001 §19 | Breakpoint trigger rules (`LineBreakpoint.Trigger`, `CommandBreakpoint` wildcards), `BreakpointUpdated`, `UpdateBreakpoints` binding algorithm, `PSDebugContext`, `CallStackFrame`. | Read `Breakpoint.cs` and `debugger.cs` L1318-L1388. | Breakpoint binding. | +| VB-055 | OPEN | RT-DEBUGGER-001 §19 | Breakpoint cmdlets, `Debug-Runspace`, `Wait-Debugger`, host prompt integration. | Read the utility cmdlets and ConsoleHost debugger code. | Debugger front end. | +| VB-056 | OPEN | RT-EVENTING-001 §19 | `OnIdle` timer logic, `UnsubscribeEvent`, `GenerateEventHandler` (IL emission), `PSEventSubscriber` members, `PSEventJob`, `PSRemoteEventManager`, `Register-ObjectEvent` parameters, `PSEventArgsCollection`. | Read `EventManager.cs` remaining ranges. | Eventing detail. | +| VB-057 | OPEN | RT-EVENTING-001 §19 | Where the `Exiting` engine event is raised and who consumes `OnScriptBlockInvoke`. | Search for `PSEngineEvent.Exiting` and `OnScriptBlockInvoke` uses. | Engine event consumers. | diff --git a/docs-out/data/canonical-facts.json b/docs-out/data/canonical-facts.json index ab4545b..1809b84 100644 --- a/docs-out/data/canonical-facts.json +++ b/docs-out/data/canonical-facts.json @@ -2,11 +2,18 @@ { "id": "CF-BND-001", "subject": "parameter-binding order and parameter-set resolution", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Parameter binding order and parameter-set resolution (CF-BND-001).** Command-line binding runs these steps in order: (1) arguments that came from hashtable splatting are moved after all other arguments; (2) a parameter name with no value is joined with the next argument as its value (a switch parameter takes `$true` instead; a name followed by another recognizable parameter name, or a name at the end, is a `MissingArgument` error); (3) **named binding**: every argument with a name is matched (exact name or alias first, then unambiguous prefix); a parameter already bound is an error (`ParameterAlreadyBound`) except that a splatted entry whose name was already bound explicitly is skipped; the `--%` verbatim marker is consumed here; (4) **positional binding**: unbound parameters that have a position in the still-valid sets are grouped by position number in ascending order; arguments without a name are taken in order; each is offered to the parameters of that position in four attempts \u2014 default set without type coercion, all valid sets without coercion, default set with coercion, all valid sets with coercion \u2014 and an argument no parameter accepts stays unbound; (5) values from `$PSDefaultParameterValues` are bound to still-unbound parameters, after positional binding and again after dynamic parameters; (6) `ValidateParameterSets`; (7) dynamic parameters are fetched once from the command (`GetDynamicParameters`), merged into the parameter table and set bitmasks, and named and positional binding are repeated for the leftover arguments; (8) leftover arguments are bound to the single `ValueFromRemainingArguments` parameter (two such parameters in the current sets is an error); (9) any argument still unbound is an error (`NamedParameterNotFound` for a named one, `PositionalParameterNotFound` otherwise); (10) the final parameter-set check and the mandatory-parameter step (\u00a77); (11) script parameters that are still unbound get their declared default values; (12) the set mask is saved for pipeline binding. **Parameter-set resolution:** each bound parameter narrows the current set mask to the sets it belongs to. After binding, with *one* valid set that set is chosen. With *several* valid sets: if the mask is \"all sets\" and the command has no default set, the call is accepted as set-less; else if the command defines a default set and it is among the valid sets, the command's parameter-set name is set to the default set, and the mask is narrowed to the default set only in validation calls that ask for it (the final check after command-line binding does; the checks made before dynamic parameters and during pipeline binding may not); else, if pipeline input is still expected and some valid set has an unbound parameter that takes pipeline input, the decision is postponed to pipeline binding; else the mask is reduced to sets with no unbound mandatory parameter; if exactly one set remains it is chosen, otherwise `AmbiguousParameterSet`. With *zero* valid sets (conflicting parameters) the result is `AmbiguousParameterSet` as well. **Pipeline-object binding** for each input object restores the saved mask and the original values of parameters bound by the previous object, then makes four passes over the unbound parameters that accept pipeline input \u2014 (a) by value without coercion, (b) by property name without coercion, (c) by value with coercion, (d) by property name with coercion \u2014 trying the default set before the other valid sets in each pass; a successful bind narrows the set mask again.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindCommandLineParametersNoValidation@L202-L277", + "src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController.BindPositionalParameters@L699-L871", + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.ValidateParameterSets@L2532-L2640", + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindPipelineParametersPrivate@L3255-L3347", + "src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L329-L667" + ], "owner": "03-runtime/parameter-binding.md", - "status": "open" + "status": "open", + "_task": "T33" }, { "id": "CF-CMP-001", @@ -70,20 +77,32 @@ { "id": "CF-DSC-001", "subject": "command resolution order", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Command resolution order (CF-DSC-001).** For one name the searcher tries, in this order and returning the first match that is allowed: (1) **alias** (a match whose visibility does not allow the caller's origin is skipped and the search continues with functions); (2) **function / filter / configuration**; (3) **cmdlet** (a module-qualified name `Module\\Cmdlet` matches only cmdlets of that module; with several cmdlets of the same name in different modules, the first in table order is returned unless all scopes are searched); (4) if application or external-script types are allowed: a name that is a *qualified provider path* is resolved as a path; then an absolute path to an existing file; then the file system search of each `PATH` directory in order (for a name with a directory part: only that directory); then a PowerShell-relative path (name starting with `.`, `~` or `\\`). Within one directory the candidate file names are, in order: the name itself first if it already has an extension; `name.ps1` when external scripts are allowed (also `.psm1` and `.psd1` in non-discovery searches); on Windows every `PATHEXT` extension when applications are allowed; then the bare name if not yet added. A `.ps1` file becomes an external script; any other existing file becomes an application. If the normal search fails and the name contains neither `-` nor `\\`, the lookup is repeated once with the name prefixed with `Get-`. If still unresolved, module-qualified names trigger module auto-loading, and (with `$PSModuleAutoLoadingPreference` = `All`, the default) other names trigger module auto-discovery against the module analysis cache; then the *command-not-found* handler runs; then a `CommandNotFoundException` is raised. In a session whose origin is `Runspace`, application and external-script types are removed from the search when the session state has no allowed applications or scripts, and a name with a directory part must match an allow-list entry (prefix match, or the list is exactly `*`).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239", + "src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.LookupCommandInfo@L656-L793", + "src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.TryNormalSearch@L886-L949", + "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.ConstructSearchPatternsFromName@L1322-L1363" + ], "owner": "03-runtime/command-discovery-and-invocation.md", - "status": "open" + "status": "open", + "_task": "T32" }, { "id": "CF-ERR-001", "subject": "terminating/non-terminating errors; try/catch/finally/trap ordering", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Terminating versus non-terminating errors; try/catch/finally/trap ordering (CF-ERR-001).** (1) A *non-terminating* error is an `ErrorRecord` written to the error stream by a command (`WriteError`); it does not interrupt the command or the pipeline, sets the pipeline's failed state and `$?` to false, and is subject to the command's `ErrorAction` (`Ignore`: dropped, not recorded; `SilentlyContinue`: recorded in `$Error`/`-ErrorVariable` but not shown; `Continue`: recorded and written; `Inquire`: ask, then as `Continue` or stop; `Stop`: converted to a terminating `ActionPreferenceStopException`). (2) A *terminating* error is an exception: `ThrowTerminatingError` from a cmdlet (wrapped as `CmdletInvocationException`), `throw`, a runtime error in an expression, or `ActionPreferenceStop`. It stops the current *statement* and, unless handled, the command, and is recorded as the pipeline's first terminating error (later ones are only logged). (3) In **script code every statement is guarded**: when a statement throws a non-flow-control exception the engine consults `$ErrorActionPreference` (default `Continue`): `SilentlyContinue`/`Ignore` resume with the next statement; `Continue` reports the error record and resumes with the next statement \u2014 except that an exception produced by a `throw` statement is rethrown when no trap is present; `Stop` rethrows (terminates the enclosing scope and propagates); `Inquire` prompts. (4) **Handler order** for one thrown exception: innermost enclosing `try` is searched first; its `catch` clauses are tried in source order with the first matching type winning (a catch without a type matches any `RuntimeException`; `PipelineStoppedException` is never caught by a type-less catch); `finally` runs after the try/catch block completes, on normal exit, after a caught exception, and while an unhandled exception propagates; if no `catch` in the `try` matches, the exception propagates to the enclosing `try`, then to `trap` handlers of the enclosing blocks (a trap matches by the exception type, trying the inner exception before the outer one), then to the statement-level preference rule of (3). A `trap` runs in a new scope with `$_` set to the error record; `continue` in a trap means *SilentlyContinue*, `break` means *Stop*; otherwise the action is the current preference. (5) Flow control (`break`, `continue`, `return`, `exit`) travels as exceptions that are never caught by `catch` or traps and are rethrown by the statement guard. (6) `$?` is true after a statement that completes without error and false after a statement in which an error (terminating-handled or non-terminating) occurred; the pipeline-level failed state also covers errors written by commands.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788", + "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968", + "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.RecordFailure@L1455-L1498", + "src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.ManageInvocationException@L848-L947" + ], "owner": "03-runtime/errors-and-diagnostics.md", - "status": "open" + "status": "open", + "_task": "T38" }, { "id": "CF-ERR-002", @@ -103,11 +122,17 @@ { "id": "CF-ETS-001", "subject": "member resolution order", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Member resolution order (CF-ETS-001).** Looking up member *name* on a value `v` (wrapped as a `PSObject`) proceeds in this order and returns the first hit: (1) the reserved names `psbase`, `psadapted`, `psextended`, `psobject` and `pstypenames`, generated on demand; (2) **instance members** added to this object (for example by `Add-Member`); (3) **type-table (extended) members** merged over the object's type-name hierarchy, most specific type name winning; (4) **adapted members** from the adapter chosen for the object; (5) **base members**: the .NET members of the wrapped object, only when the chosen adapter is not already the .NET adapter. Member-name matching in the .NET adapter is case-insensitive (its binding flags include `IgnoreCase`). Type-table collection entries are marked \"replicate and clone when returning\" (a copy bound to the current object is handed out), adapted and base entries are not; the enumeration order across sources is `UNVERIFIED` (VB-038). Member lookup on a `PSMemberSet` owner searches the set's own members first, then the same collections.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152", + "src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMappedAdapter@L460-L538", + "src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041", + "src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMemberCollection@L261-L309" + ], "owner": "03-runtime/extended-type-system.md", - "status": "open" + "status": "open", + "_task": "T35" }, { "id": "CF-FLW-001", @@ -135,12 +160,19 @@ }, { "id": "CF-LCY-001", - "subject": "command and script lifecycle (begin/process/end/clean, cmdlet lifecycle, stop)", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "subject": "command and script lifecycle (begin/process/end/clean, cmdlet lifecycle)", + "statement": "**Command lifecycle (CF-LCY-001).** For a pipeline of commands C1\u2026Cn the engine (1) builds all command processors and wires pipes; (2) *prepares* every command in order (parameter binding from the command line, with default parameter values applied) and sets up common-parameter variables; (3) runs the *begin* phase of every command in order C1\u2026Cn (`BeginProcessing`, or the `begin` block); (4) runs the *process* phase *depth-first and synchronously*: C1 reads its input (or runs once with no input if none is expected), and each object it writes into its output pipe causes C2 to run its process phase over everything queued (the default buffer threshold is zero), recursively; every input object is bound before each call (pipeline parameter binding, then mandatory-parameter check); an object that cannot be bound or misses a mandatory parameter produces a non-terminating error and is skipped; (5) when C1's input is exhausted the *end* phase runs for C1\u2026Cn in order \u2014 each command first drains any input still queued through its process phase, then runs `EndProcessing` (or the `end` block); (6) the *clean* phase (`clean` blocks, only for commands that have one) runs for every command in `finally`, with stopping suspended, and its errors are written to the command's error output; (7) all command processors are disposed. The first terminating error is recorded; later failures are logged as health events and do not replace it. A stop request cancels the pipeline token and calls `StopProcessing` on every command.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147", + "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.DoCompleteCore@L614-L709", + "src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466", + "src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Read@L452-L581", + "src/System.Management.Automation/engine/CommandProcessorBase.cs::CommandProcessorBase.Complete@L591-L609" + ], "owner": "03-runtime/runspace-and-pipeline.md", - "status": "open" + "status": "closed", + "_task": "T31" }, { "id": "CF-LEX-001", @@ -174,11 +206,18 @@ { "id": "CF-MOD-001", "subject": "module load, export and auto-load rules", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Module load, export and auto-load rules (CF-MOD-001).** (1) A module name without a path is searched in each `PSModulePath` entry in order (entries marked to skip the system folder are skipped; on Unix the folder name is matched case-insensitively): in an entry, the module base is `/`; its version sub-folders are tried first (the first whose version satisfies the constraints and equals its manifest's `ModuleVersion` is loaded); if none loads, the files `//` + extension are tried in the order `.psd1`, `.psm1`, `.cdxml`, `.ni.dll`, `.dll`, `.exe` (a name that already carries a module extension is tried as given); the first successful load ends the search and its exports are added to the analysis cache. (2) A manifest is read as data and its keys are checked against the known key list (\u00a74); the processing of the root module, nested modules, required modules (version-constrained; a cyclic-requirement check exists), required assemblies, scripts to process, types and formats follows, in an order that is `UNVERIFIED` (VB-046); how the manifest export lists combine with a script module's own exports is `UNVERIFIED` (VB-046). (3) A module already loaded that satisfies the requested constraints is not loaded again unless `-Force`; only its members are re-imported with the requested prefix and options. (4) Each module has its own session state; exported commands are copied into the importer's session state. (5) `Export-ModuleMember` selects exports by wildcard patterns per category (functions, cmdlets, variables, aliases), skipping `AllScope` items and built-in variable names. (6) Import applies `-Prefix`/`DefaultCommandPrefix`, `-NoClobber`, visibility and the verb/noun checks, in that order: cmdlets, aliases, functions, variables. (7) **Auto-loading** happens during command lookup: a module-qualified name loads exactly that module; any other unknown command name is looked up in the analysis cache of every module file on `PSModulePath` and the first module exporting a command of that exact name is imported (`$PSModuleAutoLoadingPreference`: `None`, `ModuleQualified`, `All`). (8) Removal of a module (`RemoveModule`) is `UNVERIFIED` here (VB-046).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingModulePath@L343-L423", + "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingExtensions@L5422-L5518", + "src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.ExportModuleMembers@L1529-L1689", + "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.ImportModuleMembers@L7022-L7251", + "src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCache.GetExportedCommands@L44-L96" + ], "owner": "03-runtime/modules.md", - "status": "open" + "status": "open", + "_task": "T37" }, { "id": "CF-OP-001", diff --git a/docs-out/data/claims/T3.jsonl b/docs-out/data/claims/T3.jsonl new file mode 100644 index 0000000..73b079a --- /dev/null +++ b/docs-out/data/claims/T3.jsonl @@ -0,0 +1,43 @@ +{"id": "CLM-PIP-0001", "subject_key": "pipeline.pipe.default_out_buffer_count", "predicate": "default OutBufferCount of a Pipe", "value": "0", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Pipe.cs::Pipe.OutBufferCount@L110-L110"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0002", "subject_key": "pipeline.pipe.add_order", "predicate": "order of sinks checked by Pipe.AddToPipe", "value": "nested PipelineProcessor, result collection, result list, external writer, queue", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0003", "subject_key": "pipeline.pipe.retrieve_order", "predicate": "order of sources read by Pipe.Retrieve", "value": "queue, enumerator, external reader", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Pipe.cs::Pipe.Retrieve@L541-L601"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0004", "subject_key": "pipeline.start.binding_before_begin", "predicate": "all commands are prepared (bound) before any begin phase", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147"], "doc": "RT-PIPELINE-001", "cf": "CF-LCY-001"} +{"id": "CLM-PIP-0005", "subject_key": "pipeline.first_error", "predicate": "later terminating failures replace the first recorded error", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.RecordFailure@L1455-L1498"], "doc": "RT-PIPELINE-001", "cf": "CF-LCY-001"} +{"id": "CLM-PIP-0006", "subject_key": "pipeline.thread.stack_size_bytes", "predicate": "DefaultPipelineStackSize", "value": "10000000", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline@L21-L1143"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0007", "subject_key": "pipeline.state.pipeline_state_members", "predicate": "PipelineState members in order", "value": "NotStarted, Running, Stopping, Stopped, Completed, Failed, Disconnected", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/Pipeline.cs::PipelineState@L132-L162"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-PIP-0008", "subject_key": "pipeline.concurrency.nested_must_be_sync", "predicate": "nested pipelines must be invoked synchronously", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.DoConcurrentCheck@L617-L700"], "doc": "RT-PIPELINE-001", "cf": null} +{"id": "CLM-DSC-0009", "subject_key": "discovery.order", "predicate": "search order of CommandSearcher", "value": "alias, function, cmdlet, qualified provider path, rooted path, PATH search, relative path", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239"], "doc": "RT-DISCOVERY-001", "cf": "CF-DSC-001"} +{"id": "CLM-DSC-0010", "subject_key": "discovery.get_prefix_retry", "predicate": "name without dash or backslash is retried with Get- prefix", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.TryNormalSearch@L886-L949"], "doc": "RT-DISCOVERY-001", "cf": "CF-DSC-001"} +{"id": "CLM-DSC-0011", "subject_key": "discovery.command_types.workflow_member", "predicate": "CommandTypes has a Workflow member", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommandInfo.cs::CommandTypes@L23-L76"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-DSC-0012", "subject_key": "discovery.native.process_start_phase", "predicate": "native process is started in", "value": "Prepare (before any begin phase)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Prepare@L557-L579"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-DSC-0013", "subject_key": "discovery.native.arg_passing_default", "predicate": "default value of the native argument passing variable", "value": "Windows on Windows, Standard on Unix", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.GetPassingStyle@L4675-L4682"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-DSC-0014", "subject_key": "discovery.native.exit_code_error_requires_preference", "predicate": "non-zero exit code raises an error only when PSNativeCommandUseErrorActionPreference is true", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237"], "doc": "RT-DISCOVERY-001", "cf": null} +{"id": "CLM-BND-0015", "subject_key": "binding.positional.attempt_order", "predicate": "positional binding attempts per argument", "value": "default set no coercion, all sets no coercion, default set coercion, all sets coercion", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController.BindPositionalParameters@L699-L871"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-BND-0016", "subject_key": "binding.pipeline.pass_order", "predicate": "pipeline binding passes", "value": "ValueFromPipeline no coercion, ByPropertyName no coercion, ValueFromPipeline coercion, ByPropertyName coercion", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindPipelineParametersPrivate@L3255-L3347"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-BND-0017", "subject_key": "binding.parameter_sets.max", "predicate": "maximum parameter sets per command", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.MaxParameterSetCount@L171-L171"], "doc": "RT-BIND-001", "cf": null} +{"id": "CLM-BND-0018", "subject_key": "binding.name_match", "predicate": "parameter name matching", "value": "case-insensitive unambiguous prefix over names and aliases; exact match wins", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.GetMatchingParameter@L445-L578"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-BND-0019", "subject_key": "binding.value.order", "predicate": "steps when binding one value", "value": "transformation, coercion, PSTypeName check, validation, mandatory null check, set", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L329-L667"], "doc": "RT-BIND-001", "cf": "CF-BND-001"} +{"id": "CLM-CMD-0020", "subject_key": "cmdlet.common_parameters.count", "predicate": "number of common parameters on CommonParameters", "value": "12", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/CommonCommandParameters.cs::CommonParameters@L13-L283"], "doc": "RT-CMDLET-001", "cf": null} +{"id": "CLM-CMD-0021", "subject_key": "cmdlet.write_permission", "predicate": "writes allowed only inside AllowThisCommandToWrite windows on the recorded thread", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.ThrowIfWriteNotPermitted@L2325-L2341"], "doc": "RT-CMDLET-001", "cf": null} +{"id": "CLM-CMD-0022", "subject_key": "cmdlet.shouldprocess.order", "predicate": "ShouldProcess decision order", "value": "remembered answer, WhatIf, impact threshold, prompt", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.DoShouldProcess@L1515-L1621"], "doc": "RT-CMDLET-001", "cf": null} +{"id": "CLM-PRV-0023", "subject_key": "provider.instance_per_operation", "predicate": "provider instances are created per operation", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo.CreateInstance@L546-L638"], "doc": "RT-PROVIDER-001", "cf": null} +{"id": "CLM-PRV-0024", "subject_key": "provider.hierarchy", "predicate": "provider class hierarchy", "value": "CmdletProvider, DriveCmdletProvider, ItemCmdletProvider, ContainerCmdletProvider, NavigationCmdletProvider", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo@L22-L687"], "doc": "RT-PROVIDER-001", "cf": null} +{"id": "CLM-ETS-0025", "subject_key": "ets.adapter_lookup", "predicate": "type-table adapter lookup key", "value": "exact full type name", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/TypeTable.cs::TypeTable.GetTypeAdapter@L4064-L4099"], "doc": "RT-ETS-001", "cf": "CF-ETS-001"} +{"id": "CLM-ETS-0026", "subject_key": "ets.member_order", "predicate": "member lookup order", "value": "reserved names, instance, type table, adapter, base", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152"], "doc": "RT-ETS-001", "cf": "CF-ETS-001"} +{"id": "CLM-ETS-0027", "subject_key": "ets.typetable.merge_direction", "predicate": "type-table members merged from", "value": "least specific to most specific type name", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041"], "doc": "RT-ETS-001", "cf": "CF-ETS-001"} +{"id": "CLM-FMT-0028", "subject_key": "format.table_cache_window_ms", "predicate": "auto-width table caching window", "value": "300", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394"], "doc": "RT-FORMAT-001", "cf": null} +{"id": "CLM-FMT-0029", "subject_key": "format.fallback_shape", "predicate": "shape when no view and property count at or below table threshold", "value": "Table", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataQuery.cs::DisplayDataQuery.GetShapeFromPropertyCount@L333-L339"], "doc": "RT-FORMAT-001", "cf": null} +{"id": "CLM-HLP-0030", "subject_key": "help.provider_order", "predicate": "help provider chain order", "value": "alias, script command, command, provider, class, help file, default", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.InitializeHelpProviders@L563-L593"], "doc": "RT-HELP-001", "cf": null} +{"id": "CLM-MOD-0031", "subject_key": "modules.extension_order", "predicate": "implicit module file extension order", "value": ".psd1, .psm1, .cdxml, .ni.dll, .dll, .exe", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.PSModuleExtensions@L896-L905"], "doc": "RT-MODULES-001", "cf": "CF-MOD-001"} +{"id": "CLM-MOD-0032", "subject_key": "modules.version_folder_rule", "predicate": "version folder accepted only if manifest ModuleVersion equals folder version", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingMultiVersionModuleBase@L433-L478"], "doc": "RT-MODULES-001", "cf": "CF-MOD-001"} +{"id": "CLM-MOD-0033", "subject_key": "modules.analysis_cache_flush_delay_ms", "predicate": "delay before the analysis cache is serialized", "value": "10000", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCacheData@L645-L1131"], "doc": "RT-MODULES-001", "cf": null} +{"id": "CLM-ERR-0034", "subject_key": "errors.dollar_error_max", "predicate": "maximum entries kept in $Error", "value": "256", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L818-L858"], "doc": "RT-ERRORS-001", "cf": null} +{"id": "CLM-ERR-0035", "subject_key": "errors.error_category.count", "predicate": "number of ErrorCategory members", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ErrorPackage.cs::ErrorCategory@L28-L189"], "doc": "RT-ERRORS-001", "cf": null} +{"id": "CLM-ERR-0036", "subject_key": "errors.question_mark", "predicate": "$? is read-only and all-scope", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/QuestionMarkVariable.cs::QuestionMarkVariable@L9-L42"], "doc": "RT-ERRORS-001", "cf": null} +{"id": "CLM-JOB-0037", "subject_key": "jobs.foreach_parallel.default_throttle", "predicate": "default ThrottleLimit of ForEach-Object -Parallel", "value": "5", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/InternalCommands.cs::ForEachObjectCommand@L65-L1271"], "doc": "RT-JOBS-001", "cf": null} +{"id": "CLM-JOB-0038", "subject_key": "jobs.job_states.count", "predicate": "number of JobState members", "value": "11", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/remoting/client/Job.cs::JobState@L31-L90"], "doc": "RT-JOBS-001", "cf": null} +{"id": "CLM-JOB-0039", "subject_key": "jobs.background.child_args", "predicate": "child pwsh arguments of a background job", "value": "-s -NoLogo -NoProfile", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs::PowerShellProcessInstance@L16-L292"], "doc": "RT-JOBS-001", "cf": null} +{"id": "CLM-DBG-0040", "subject_key": "debugger.resume_actions", "predicate": "DebuggerResumeAction members", "value": "Continue, StepInto, StepOut, StepOver, Stop", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerResumeAction@L31-L53"], "doc": "RT-DEBUGGER-001", "cf": null} +{"id": "CLM-DBG-0041", "subject_key": "debugger.prompt_commands", "predicate": "debugger prompt commands", "value": "continue c, stepInto s, stepOver v, stepOut o, quit q, detach d, k, list l, h", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/debugger/debugger.cs::DebuggerCommandProcessor@L5085-L5419"], "doc": "RT-DEBUGGER-001", "cf": null} +{"id": "CLM-EVT-0042", "subject_key": "events.reserved_prefix", "predicate": "source identifiers reserved for the engine start with", "value": "PowerShell.", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewSubscriber@L645-L796"], "doc": "RT-EVENTING-001", "cf": null} +{"id": "CLM-EVT-0043", "subject_key": "events.action_throttle", "predicate": "actions run per pulse", "value": "1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager@L321-L1546"], "doc": "RT-EVENTING-001", "cf": null} diff --git a/docs-out/data/lessons.json b/docs-out/data/lessons.json index 36e374d..db7e981 100644 --- a/docs-out/data/lessons.json +++ b/docs-out/data/lessons.json @@ -153,6 +153,347 @@ "file": "01-repository/coding-standards.md", "line": 111 }, + { + "id": "LSN-RTCMD-01", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Keep the four hooks (begin, process, end, stop) plus clean, with `Stopping` checked on every write.", + "evidence": "src/System.Management.Automation/engine/cmdlet.cs::Cmdlet.DoBeginProcessing@L141-L166.", + "dev": null, + "doc": "RT-CMDLET-001", + "file": "03-runtime/cmdlet-infrastructure.md", + "line": 162 + }, + { + "id": "LSN-RTCMD-02", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Separate typed streams with per-stream action preferences and the Ignore / SilentlyContinue / Continue / Inquire / Stop semantics.", + "evidence": "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968.", + "dev": null, + "doc": "RT-CMDLET-001", + "file": "03-runtime/cmdlet-infrastructure.md", + "line": 163 + }, + { + "id": "LSN-RTCMD-03", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "`ShouldProcess` decision order (remembered answer \u2192 what-if \u2192 impact threshold \u2192 prompt).", + "evidence": "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.DoShouldProcess@L1515-L1621.", + "dev": null, + "doc": "RT-CMDLET-001", + "file": "03-runtime/cmdlet-infrastructure.md", + "line": 164 + }, + { + "id": "LSN-RTCMD-04", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "Make write permission a capability object passed to the cmdlet for the duration of a phase instead of three mutable fields on the pipeline processor checked against the current thread.", + "evidence": "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime.AllowThisCommandToWrite@L2349-L2352.", + "dev": null, + "doc": "RT-CMDLET-001", + "file": "03-runtime/cmdlet-infrastructure.md", + "line": 165 + }, + { + "id": "LSN-RTCMD-05", + "verdict": "ADAPT", + "area": "NLD-CMD", + "text": "Split the 3,800-line runtime into stream writers (one per stream) behind one interface, keeping behavior identical.", + "evidence": "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime@L29-L3829.", + "dev": null, + "doc": "RT-CMDLET-001", + "file": "03-runtime/cmdlet-infrastructure.md", + "line": 166 + }, + { + "id": "LSN-RTCMD-06", + "verdict": "AVOID", + "area": "NLD-CMD", + "text": "Do not carry transaction support (`-UseTransaction`, `SupportsTransactions`, the engine protection scope) into the new shell; the feature is Windows-only and out of core scope. DEV: DEV-CMDLET-135.", + "evidence": "src/System.Management.Automation/engine/cmdlet.cs::Cmdlet.DoBeginProcessing@L141-L166.", + "dev": "DEV-CMDLET-135", + "doc": "RT-CMDLET-001", + "file": "03-runtime/cmdlet-infrastructure.md", + "line": 167 + }, + { + "id": "LSN-RTDISC-01", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Resolution order alias \u2192 function \u2192 cmdlet \u2192 path/file, with module-qualified names bypassing precedence; it is observable language behavior.", + "evidence": "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.MoveNext@L78-L239.", + "dev": null, + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 192 + }, + { + "id": "LSN-RTDISC-02", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Make the searcher a resumable enumerator so `Get-Command`-style wildcard queries share the code with single lookups.", + "evidence": "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher@L22-L1708.", + "dev": null, + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 193 + }, + { + "id": "LSN-RTDISC-03", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "Keep the lookup events (pre, not-found, post) but give them typed results and a re-entrancy guard in the type system instead of a name\u2192set dictionary of strings.", + "evidence": "src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.RegisterLookupCommandInfoAction@L1156-L1169.", + "dev": null, + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 194 + }, + { + "id": "LSN-RTDISC-04", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Visibility and trust-level filtering belong in the lookup, not only at run time.", + "evidence": "src/System.Management.Automation/engine/CommandSearcher.cs::CommandSearcher.ShouldSkipCommandResolutionForConstrainedLanguage@L836-L876.", + "dev": null, + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 195 + }, + { + "id": "LSN-RTDISC-05", + "verdict": "REDESIGN", + "area": "NLD-CMD", + "text": "Replace module auto-discovery by file analysis on every unknown name with an explicit, signed command index that is refreshed on module install/update; keep module-qualified auto-load. DEV: DEV-DISC-120.", + "evidence": "src/System.Management.Automation/engine/CommandDiscovery.cs::CommandDiscovery.TryModuleAutoDiscovery@L951-L1055.", + "dev": "DEV-DISC-120", + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 196 + }, + { + "id": "LSN-RTDISC-06", + "verdict": "AVOID", + "area": "NLD-CMD", + "text": "Do not implement the `Legacy` native-argument passing style (string re-quoting) or the name-based legacy-program list; offer only list-based passing (`Standard`) and a per-command opt-in for programs that need a raw command line. DEV: DEV-DISC-121.", + "evidence": "src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.ShouldUseLegacyPassingStyle@L1720-L1729.", + "dev": "DEV-DISC-121", + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 197 + }, + { + "id": "LSN-RTDISC-07", + "verdict": "ADAPT", + "area": "NLD-CMD", + "text": "Start the child process after all begin phases (or make the start point explicit) so pipeline ordering is the same for cmdlets and programs; keep the \"console-direct when last into the default output\" optimization.", + "evidence": "src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Prepare@L557-L579.", + "dev": null, + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 198 + }, + { + "id": "LSN-RTDISC-08", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "`$LASTEXITCODE` set on every native exit; error-on-non-zero as an opt-in preference.", + "evidence": "src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237.", + "dev": null, + "doc": "RT-DISCOVERY-001", + "file": "03-runtime/command-discovery-and-invocation.md", + "line": 199 + }, + { + "id": "LSN-RTDBG-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Sequence points with per-function bit arrays and a single hook called only in debug mode.", + "evidence": "src/System.Management.Automation/engine/debugger/debugger.cs::ScriptDebugger.OnSequencePointHit@L1537-L1604.", + "dev": null, + "doc": "RT-DEBUGGER-001", + "file": "03-runtime/debugger.md", + "line": 173 + }, + { + "id": "LSN-RTDBG-02", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Three breakpoint kinds (line, command, variable) with action blocks, and the five resume actions.", + "evidence": "src/System.Management.Automation/engine/debugger/Breakpoint.cs::Breakpoint@L16-L150.", + "dev": null, + "doc": "RT-DEBUGGER-001", + "file": "03-runtime/debugger.md", + "line": 174 + }, + { + "id": "LSN-RTDBG-03", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Expose the debugger through a protocol-neutral interface suited to an external adapter (the Debug Adapter Protocol), keeping the prompt commands for terminals.", + "evidence": "src/System.Management.Automation/engine/debugger/debugger.cs::Debugger@L320-L966.", + "dev": null, + "doc": "RT-DEBUGGER-001", + "file": "03-runtime/debugger.md", + "line": 175 + }, + { + "id": "LSN-RTDBG-04", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Debugger attach/enable is a policy-controlled, audited privilege, and the prompt runs under the lowest trust level present on the stack. DEV: DEV-DBG-195.", + "evidence": "src/System.Management.Automation/engine/debugger/debugger.cs::NestedRunspaceDebugger@L4227-L4629.", + "dev": "DEV-DBG-195", + "doc": "RT-DEBUGGER-001", + "file": "03-runtime/debugger.md", + "line": 176 + }, + { + "id": "LSN-RTDBG-05", + "verdict": "AVOID", + "area": "NLD-TOOL", + "text": "Do not port nested debugging of jobs and remote runspaces (`DebugJob`, `Embedded/StandaloneRunspaceDebugger`) in the first version. DEV: DEV-DBG-196.", + "evidence": "src/System.Management.Automation/engine/debugger/debugger.cs::StandaloneRunspaceDebugger@L4636-L4729.", + "dev": "DEV-DBG-196", + "doc": "RT-DEBUGGER-001", + "file": "03-runtime/debugger.md", + "line": 177 + }, + { + "id": "LSN-RTERR-01", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Split errors into records (non-terminating, streamed) and exceptions (terminating), with the `ErrorAction` decision list; it defines script behavior.", + "evidence": "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968.", + "dev": null, + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 179 + }, + { + "id": "LSN-RTERR-02", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Statement-level guard with continue-by-default and `throw` as the always-terminating form; keep traps with `continue`/`break` semantics.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788.", + "dev": null, + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 180 + }, + { + "id": "LSN-RTERR-03", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "`$Error` as a bounded, newest-first, de-duplicated history; `$?` as a read-only all-scope variable.", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L818-L858.", + "dev": null, + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 181 + }, + { + "id": "LSN-RTERR-04", + "verdict": "ADAPT", + "area": "NLD-ERR", + "text": "Keep `ErrorRecord` fields and the 32 categories but add a schema version, a correlation id and a redaction marker for sensitive fields.", + "evidence": "src/System.Management.Automation/engine/ErrorPackage.cs::ErrorRecord@L988-L1699.", + "dev": null, + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 182 + }, + { + "id": "LSN-RTERR-05", + "verdict": "ADAPT", + "area": "NLD-ERR", + "text": "Make the `$Error` capacity configurable (the source hard-codes 256 and no preference variable controls it).", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.AppendDollarError@L847-L855.", + "dev": null, + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 183 + }, + { + "id": "LSN-RTERR-06", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Native exit codes in `$LASTEXITCODE` and `$?`, error on non-zero only by preference.", + "evidence": "src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.Complete@L1082-L1237.", + "dev": null, + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 184 + }, + { + "id": "LSN-RTERR-07", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Separate the audit stream (policy decisions, denials) from the user error stream so that `-ErrorAction SilentlyContinue` or `Ignore` can never suppress an audit record. DEV: DEV-ERR-180.", + "evidence": "src/System.Management.Automation/engine/MshCommandRuntime.cs::MshCommandRuntime._WriteErrorSkipAllowCheck@L2858-L2968.", + "dev": "DEV-ERR-180", + "doc": "RT-ERRORS-001", + "file": "03-runtime/errors-and-diagnostics.md", + "line": 185 + }, + { + "id": "LSN-RTEVT-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Source-identifier based events with queue, actions, auto-unregister and reserved engine identifiers.", + "evidence": "src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewEventImplementation@L957-L1018.", + "dev": null, + "doc": "RT-EVENTING-001", + "file": "03-runtime/eventing.md", + "line": 151 + }, + { + "id": "LSN-RTEVT-02", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Run actions serially at idle points in the subscriber's session state.", + "evidence": "src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.InvokeAction@L1222-L1281.", + "dev": null, + "doc": "RT-EVENTING-001", + "file": "03-runtime/eventing.md", + "line": 152 + }, + { + "id": "LSN-RTEVT-03", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Replace `Reflection.Emit` handler generation with expression-tree or `Delegate.CreateDelegate` adapters (or `DynamicInvoke` fallback) so the feature works where dynamic assemblies are blocked.", + "evidence": "src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessNewSubscriber@L645-L796.", + "dev": null, + "doc": "RT-EVENTING-001", + "file": "03-runtime/eventing.md", + "line": 153 + }, + { + "id": "LSN-RTEVT-04", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Bound the received-event queue and action queue with policy-defined limits and drop/oldest rules. DEV: DEV-EVT-198.", + "evidence": "src/System.Management.Automation/engine/EventManager.cs::PSLocalEventManager.ProcessPendingActionsImpl@L1072-L1137.", + "dev": "DEV-EVT-198", + "doc": "RT-EVENTING-001", + "file": "03-runtime/eventing.md", + "line": 154 + }, + { + "id": "LSN-RTEVT-05", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not port forwarded events (`PSRemoteEventManager`) or the WMI/CIM event sources in the first version. DEV: DEV-EVT-199.", + "evidence": "src/System.Management.Automation/engine/EventManager.cs::PSRemoteEventManager@L1551-L1811.", + "dev": "DEV-EVT-199", + "doc": "RT-EVENTING-001", + "file": "03-runtime/eventing.md", + "line": 155 + }, { "id": "LSN-CTX-01", "verdict": "REPLICATE", @@ -197,6 +538,567 @@ "file": "03-runtime/execution-context-and-session-state.md", "line": 118 }, + { + "id": "LSN-RTETS-01", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Keep the member resolution order instance \u2192 type data \u2192 adapter \u2192 base with the five reserved names; scripts depend on it.", + "evidence": "src/System.Management.Automation/engine/MshMemberInfo.cs::PSMemberInfoIntegratingCollection@L4503-L5152.", + "dev": null, + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 167 + }, + { + "id": "LSN-RTETS-02", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Merge type data from the least to the most specific type name, with member-set inheritance.", + "evidence": "src/System.Management.Automation/engine/TypeTable.cs::TypeTable.MemberFactory@L3985-L4041.", + "dev": null, + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 168 + }, + { + "id": "LSN-RTETS-03", + "verdict": "ADAPT", + "area": "NLD-TYP", + "text": "Select adapters by type with inheritance (nearest registered base type wins) instead of exact full-name match, as the source comment itself proposes.", + "evidence": "src/System.Management.Automation/engine/TypeTable.cs::TypeTable.GetTypeAdapter@L4064-L4099.", + "dev": null, + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 169 + }, + { + "id": "LSN-RTETS-04", + "verdict": "ADAPT", + "area": "NLD-TYP", + "text": "Make adapter and type-table registries per-runspace (or per-session-state) objects rather than static caches, so one session cannot change another's member view.", + "evidence": "src/System.Management.Automation/engine/MshObject.cs::PSObject.GetMappedAdapter@L460-L538.", + "dev": null, + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 170 + }, + { + "id": "LSN-RTETS-05", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Bind script members to the definer's language mode and keep deserialized objects data-only.", + "evidence": "src/System.Management.Automation/engine/MshMemberInfo.cs::PSScriptProperty@L1537-L1905.", + "dev": null, + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 171 + }, + { + "id": "LSN-RTETS-06", + "verdict": "AVOID", + "area": "NLD-TYP", + "text": "Do not carry the WMI, DirectoryEntry, DataRow/DataRowView and COM adapters into the first version; keep .NET, XML, `PSObject`, member-set and the third-party adapter bridge. DEV: DEV-ETS-150.", + "evidence": "src/System.Management.Automation/engine/MshObject.cs::PSObject.MappedInternalAdapterSet@L433-L454.", + "dev": "DEV-ETS-150", + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 172 + }, + { + "id": "LSN-RTETS-07", + "verdict": "REDESIGN", + "area": "NLD-TYP", + "text": "Compile the built-in type data to a table at build time (as PowerShell does) but also publish it as a versioned data file, so tooling and a clean-room implementation share one source. DEV: DEV-ETS-151.", + "evidence": "src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs::TypeTable@L12-L9138.", + "dev": "DEV-ETS-151", + "doc": "RT-ETS-001", + "file": "03-runtime/extended-type-system.md", + "line": 173 + }, + { + "id": "LSN-RTFMT-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Two-stage output: objects \u2192 formatting records (data) \u2192 line layout, with the record state machine; `Format-*` output can be piped to `Out-*`.", + "evidence": "src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394.", + "dev": null, + "doc": "RT-FORMAT-001", + "file": "03-runtime/formatting-and-output.md", + "line": 190 + }, + { + "id": "LSN-RTFMT-02", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "View selection by type-name list with perfect match then best match, then property-count fallback (table at or below the threshold, else list).", + "evidence": "src/System.Management.Automation/FormatAndOutput/common/FormatViewManager.cs::FormatViewManager.Initialize@L91-L204.", + "dev": null, + "doc": "RT-FORMAT-001", + "file": "03-runtime/formatting-and-output.md", + "line": 191 + }, + { + "id": "LSN-RTFMT-03", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Replace the hidden sub-pipeline (`CommandWrapper` over `Out-LineOutput`) by a direct call to the formatter and sink objects, keeping the same observable output.", + "evidence": "src/System.Management.Automation/FormatAndOutput/common/OutputManager.cs::SubPipelineManager@L141-L326.", + "dev": null, + "doc": "RT-FORMAT-001", + "file": "03-runtime/formatting-and-output.md", + "line": 192 + }, + { + "id": "LSN-RTFMT-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Keep table width auto-sizing but make the buffering window a configurable policy (count and time) instead of the fixed 300 ms.", + "evidence": "src/System.Management.Automation/FormatAndOutput/common/BaseOutputtingCommand.cs::OutCommandInner@L16-L1394.", + "dev": null, + "doc": "RT-FORMAT-001", + "file": "03-runtime/formatting-and-output.md", + "line": 193 + }, + { + "id": "LSN-RTFMT-05", + "verdict": "REDESIGN", + "area": "NLD-RUN", + "text": "Provide an optional structured output sink (typed records for machines) next to the text sink, so enterprise tooling does not parse text. DEV: DEV-FMT-160.", + "evidence": "src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs::LineOutput@L215-L304.", + "dev": "DEV-FMT-160", + "doc": "RT-FORMAT-001", + "file": "03-runtime/formatting-and-output.md", + "line": 194 + }, + { + "id": "LSN-RTFMT-06", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not carry the format XML loader family (`typeDataXmlLoader*`, schema validation) in the first version; ship built-in views as compiled data plus a small JSON/YAML view definition for extensions. DEV: DEV-FMT-161.", + "evidence": "src/System.Management.Automation/FormatAndOutput/common/DisplayDatabase/typeDataXmlLoader.cs::TypeInfoDataBaseLoader@L40-L2234.", + "dev": "DEV-FMT-161", + "doc": "RT-FORMAT-001", + "file": "03-runtime/formatting-and-output.md", + "line": 195 + }, + { + "id": "LSN-RTHELP-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Comment-based help with the keyword set above and `.EXTERNALHELP`/forwarding, since scripts written for PowerShell carry it.", + "evidence": "src/System.Management.Automation/help/HelpCommentsParser.cs::HelpCommentsParser.AnalyzeCommentBlock@L712-L808.", + "dev": null, + "doc": "RT-HELP-001", + "file": "03-runtime/help-system.md", + "line": 164 + }, + { + "id": "LSN-RTHELP-02", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Replace the fixed provider list with registered providers that declare category and priority, keeping the same default order.", + "evidence": "src/System.Management.Automation/help/HelpSystem.cs::HelpSystem.InitializeHelpProviders@L563-L593.", + "dev": null, + "doc": "RT-HELP-001", + "file": "03-runtime/help-system.md", + "line": 165 + }, + { + "id": "LSN-RTHELP-03", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Keep forwarding but make cycles a static check at load rather than a run-time exception.", + "evidence": "src/System.Management.Automation/help/CommandHelpProvider.cs::CommandHelpProvider.ExactMatchHelp@L425-L472.", + "dev": null, + "doc": "RT-HELP-001", + "file": "03-runtime/help-system.md", + "line": 166 + }, + { + "id": "LSN-RTHELP-04", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Deliver updatable help as signed packages with a catalog and an offline mode, verified before install. DEV: DEV-HELP-165.", + "evidence": "src/System.Management.Automation/help/UpdatableHelpSystem.cs::UpdatableHelpSystem@L233-L1573.", + "dev": "DEV-HELP-165", + "doc": "RT-HELP-001", + "file": "03-runtime/help-system.md", + "line": 167 + }, + { + "id": "LSN-RTHELP-05", + "verdict": "AVOID", + "area": "NLD-TOOL", + "text": "Do not port MAML as the authoring format or the DSC resource provider; use a lighter structured format (and the comment-based format) for built-in help. DEV: DEV-HELP-166.", + "evidence": "src/System.Management.Automation/help/MamlNode.cs::MamlNode@L67-L1251.", + "dev": "DEV-HELP-166", + "doc": "RT-HELP-001", + "file": "03-runtime/help-system.md", + "line": 168 + }, + { + "id": "LSN-RTJOB-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "One `Job` abstraction with child jobs, per-stream collections, finished-state semantics and a state-changed event.", + "evidence": "src/System.Management.Automation/engine/remoting/client/Job.cs::Job.SetJobState@L1386-L1462.", + "dev": null, + "doc": "RT-JOBS-001", + "file": "03-runtime/jobs-and-concurrency.md", + "line": 177 + }, + { + "id": "LSN-RTJOB-02", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "`ForEach-Object -Parallel` semantics: throttle default 5, `$using:` capture, completion-order output, timeout.", + "evidence": "src/System.Management.Automation/engine/InternalCommands.cs::ForEachObjectCommand@L65-L1271.", + "dev": null, + "doc": "RT-JOBS-001", + "file": "03-runtime/jobs-and-concurrency.md", + "line": 178 + }, + { + "id": "LSN-RTJOB-03", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Add an explicit transition table to the job state machine (the source accepts any transition).", + "evidence": "src/System.Management.Automation/engine/remoting/client/Job.cs::Job.SetJobState@L1386-L1462.", + "dev": null, + "doc": "RT-JOBS-001", + "file": "03-runtime/jobs-and-concurrency.md", + "line": 179 + }, + { + "id": "LSN-RTJOB-04", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Child processes and parallel runspaces inherit policy, language mode and visibility from the parent as a hard rule, plus per-job quotas. DEV: DEV-JOB-190.", + "evidence": "src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs::PowerShellProcessInstance@L16-L292.", + "dev": "DEV-JOB-190", + "doc": "RT-JOBS-001", + "file": "03-runtime/jobs-and-concurrency.md", + "line": 180 + }, + { + "id": "LSN-RTJOB-05", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not port `ThrottlingJob`, `ContainerParentJob` and the job source-adapter registry in the first version; one local job type plus the parallel pool suffices. DEV: DEV-JOB-191.", + "evidence": "src/System.Management.Automation/engine/remoting/client/ThrottlingJob.cs::ThrottlingJob@L28-L1262.", + "dev": "DEV-JOB-191", + "doc": "RT-JOBS-001", + "file": "03-runtime/jobs-and-concurrency.md", + "line": 181 + }, + { + "id": "LSN-RTMOD-01", + "verdict": "REPLICATE", + "area": "NLD-MOD", + "text": "Module = folder + manifest + optional root script/binary, with per-module session state and an export list; manifest is data, not code.", + "evidence": "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingExtensions@L5422-L5518.", + "dev": null, + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 184 + }, + { + "id": "LSN-RTMOD-02", + "verdict": "REPLICATE", + "area": "NLD-MOD", + "text": "Version-folder rule (folder version equals manifest version) and constraint matching with required/minimum/maximum.", + "evidence": "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.LoadUsingMultiVersionModuleBase@L433-L478.", + "dev": null, + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 185 + }, + { + "id": "LSN-RTMOD-03", + "verdict": "REPLICATE", + "area": "NLD-MOD", + "text": "Export engine with wildcard patterns per category and manifest lists overriding script exports.", + "evidence": "src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.ExportModuleMembers@L1529-L1689.", + "dev": null, + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 186 + }, + { + "id": "LSN-RTMOD-04", + "verdict": "ADAPT", + "area": "NLD-MOD", + "text": "Keep the exported-command analysis cache but make it a versioned, signed, explicitly refreshed index (see DEV-DISC-120), validated by content hash rather than last write time.", + "evidence": "src/System.Management.Automation/engine/Modules/AnalysisCache.cs::AnalysisCache.GetExportedCommands@L44-L96.", + "dev": null, + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 187 + }, + { + "id": "LSN-RTMOD-05", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Module search path entries carry trust levels, and auto-load applies only from trusted entries unless policy allows more. DEV: DEV-MOD-170.", + "evidence": "src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs::ModuleIntrinsics.GetModulePath@L1397-L1419.", + "dev": "DEV-MOD-170", + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 188 + }, + { + "id": "LSN-RTMOD-06", + "verdict": "ADAPT", + "area": "NLD-MOD", + "text": "Split the 7,500-line loader class by responsibility: path resolution, manifest reader, module factory per kind, export engine, importer, remover.", + "evidence": "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase@L40-L7478.", + "dev": null, + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 189 + }, + { + "id": "LSN-RTMOD-07", + "verdict": "AVOID", + "area": "NLD-MOD", + "text": "Do not implement Windows PowerShell compatibility imports (`ImportModulesUsingWinCompat`), `.cdxml` CIM modules or DSC resource exports in the first version. DEV: DEV-MOD-171.", + "evidence": "src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs::ModuleCmdletBase.ImportModulesUsingWinCompat@L4972-L5006.", + "dev": "DEV-MOD-171", + "doc": "RT-MODULES-001", + "file": "03-runtime/modules.md", + "line": 190 + }, + { + "id": "LSN-RTBIND-01", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Keep the binding order (named \u2192 positional \u2192 defaults \u2192 dynamic \u2192 remaining \u2192 mandatory \u2192 pipeline) and the exact \u2192 coerced two-pass for positional and pipeline binding; it is observable.", + "evidence": "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.BindCommandLineParametersNoValidation@L202-L277.", + "dev": null, + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 177 + }, + { + "id": "LSN-RTBIND-02", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Parameter sets as bit masks with a default set and the ambiguity-by-mandatory tie-break.", + "evidence": "src/System.Management.Automation/engine/CmdletParameterBinderController.cs::CmdletParameterBinderController.ValidateParameterSets@L2532-L2640.", + "dev": null, + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 178 + }, + { + "id": "LSN-RTBIND-03", + "verdict": "REPLICATE", + "area": "NLD-CMD", + "text": "Unambiguous-prefix parameter names including aliases; exact match wins.", + "evidence": "src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.GetMatchingParameter@L445-L578.", + "dev": null, + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 179 + }, + { + "id": "LSN-RTBIND-04", + "verdict": "ADAPT", + "area": "NLD-CMD", + "text": "Lift the 32-set limit by using a set-index bit vector type, keeping the same semantics.", + "evidence": "src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs::MergedCommandParameterMetadata.MaxParameterSetCount@L171-L171.", + "dev": null, + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 180 + }, + { + "id": "LSN-RTBIND-05", + "verdict": "ADAPT", + "area": "NLD-CMD", + "text": "Factor the \"default binding in use \u2192 elaborate error\" pattern into one wrapper, and the binding steps into an explicit pipeline of stages with a trace result object.", + "evidence": "src/System.Management.Automation/engine/ParameterBinderController.cs::ParameterBinderController.ThrowElaboratedBindingException@L1017-L1049.", + "dev": null, + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 181 + }, + { + "id": "LSN-RTBIND-06", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Run transformation/validation callbacks under the callee's language mode; keep the explicit transition flag.", + "evidence": "src/System.Management.Automation/engine/CommandProcessor.cs::CommandProcessor.Prepare@L193-L254.", + "dev": null, + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 182 + }, + { + "id": "LSN-RTBIND-07", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Redact bound values in execution-detail logging by parameter type and attribute instead of logging the first 256 characters of every value. DEV: DEV-BIND-130.", + "evidence": "src/System.Management.Automation/engine/ParameterBinderBase.cs::ParameterBinderBase.BindParameter@L329-L667.", + "dev": "DEV-BIND-130", + "doc": "RT-BIND-001", + "file": "03-runtime/parameter-binding.md", + "line": 183 + }, + { + "id": "LSN-RTPROV-01", + "verdict": "ADAPT", + "area": "NLD-MOD", + "text": "Keep the verb contract (item, container, navigation, content, property) as interfaces with capability flags instead of a five-level class hierarchy with context-bound state.", + "evidence": "src/System.Management.Automation/namespaces/NavigationProviderBase.cs::NavigationCmdletProvider@L21-L1088.", + "dev": null, + "doc": "RT-PROVIDER-001", + "file": "03-runtime/provider-model.md", + "line": 185 + }, + { + "id": "LSN-RTPROV-02", + "verdict": "REPLICATE", + "area": "NLD-MOD", + "text": "Drive-qualified, provider-qualified and relative path forms and per-scope drives.", + "evidence": "src/System.Management.Automation/namespaces/LocationGlobber.cs::LocationGlobber.IsProviderQualifiedPath@L1371-L1431.", + "dev": null, + "doc": "RT-PROVIDER-001", + "file": "03-runtime/provider-model.md", + "line": 186 + }, + { + "id": "LSN-RTPROV-03", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Put a mandatory policy mediation layer between the engine and providers (root confinement, rights, audit) and pass an explicit per-call context object instead of mutating a `Context` field on the provider. DEV: DEV-PROV-140.", + "evidence": "src/System.Management.Automation/engine/DataStoreAdapterProvider.cs::ProviderInfo.CreateInstance@L546-L638.", + "dev": "DEV-PROV-140", + "doc": "RT-PROVIDER-001", + "file": "03-runtime/provider-model.md", + "line": 187 + }, + { + "id": "LSN-RTPROV-04", + "verdict": "REPLICATE", + "area": "NLD-MOD", + "text": "Treat variables, functions, aliases and environment as providers over session state.", + "evidence": "src/System.Management.Automation/engine/InitialSessionState.cs::PSSnapInHelpers@L5030-L5613.", + "dev": null, + "doc": "RT-PROVIDER-001", + "file": "03-runtime/provider-model.md", + "line": 188 + }, + { + "id": "LSN-RTPROV-05", + "verdict": "ADAPT", + "area": "NLD-MOD", + "text": "Replace the all-exceptions-swallowed `ItemExists` with a typed result (exists, absent, failed) and let `Test-Path` decide.", + "evidence": "src/System.Management.Automation/namespaces/ItemProviderBase.cs::ItemCmdletProvider.ItemExists@L248-L266.", + "dev": null, + "doc": "RT-PROVIDER-001", + "file": "03-runtime/provider-model.md", + "line": 189 + }, + { + "id": "LSN-RTPROV-06", + "verdict": "AVOID", + "area": "NLD-MOD", + "text": "Do not implement file-system auto-mounting of Windows volumes on lookup miss as an engine feature; mount drives explicitly or from configuration. DEV: DEV-PROV-141.", + "evidence": "src/System.Management.Automation/engine/SessionStateDriveAPIs.cs::SessionStateInternal.GetDrive@L415-L477.", + "dev": "DEV-PROV-141", + "doc": "RT-PROVIDER-001", + "file": "03-runtime/provider-model.md", + "line": 190 + }, + { + "id": "LSN-RTPIPE-01", + "verdict": "REPLICATE", + "area": "NLD-PIP", + "text": "Keep the synchronous depth-first push model with a zero default buffer threshold: it defines the observable streaming behavior of pipelines.", + "evidence": "src/System.Management.Automation/engine/Pipe.cs::Pipe.AddToPipe@L435-L466.", + "dev": null, + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 253 + }, + { + "id": "LSN-RTPIPE-02", + "verdict": "REPLICATE", + "area": "NLD-PIP", + "text": "Bind all commands, then run all begin phases in order, then process, then end in order; keep the clean phase outside the stop path.", + "evidence": "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.Start@L1029-L1147.", + "dev": null, + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 254 + }, + { + "id": "LSN-RTPIPE-03", + "verdict": "REPLICATE", + "area": "NLD-PIP", + "text": "First terminating error wins; later failures are logged, not thrown.", + "evidence": "src/System.Management.Automation/engine/pipeline.cs::PipelineProcessor.RecordFailure@L1455-L1498.", + "dev": null, + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 255 + }, + { + "id": "LSN-RTPIPE-04", + "verdict": "ADAPT", + "area": "NLD-PIP", + "text": "Keep the one-pipeline-per-runspace rule but make the state machines explicit tables with a single transition function; the original spreads transitions over several classes with events queued under locks.", + "evidence": "src/System.Management.Automation/engine/hostifaces/pipelinebase.cs::PipelineBase.SetPipelineState@L798-L823.", + "dev": null, + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 256 + }, + { + "id": "LSN-RTPIPE-05", + "verdict": "ADAPT", + "area": "NLD-PIP", + "text": "Replace the fixed 10 MB thread stack plus per-call stack probe with a configurable depth limit that raises the same `ScriptCallDepth` error.", + "evidence": "src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline@L21-L1143.", + "dev": null, + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 257 + }, + { + "id": "LSN-RTPIPE-06", + "verdict": "REDESIGN", + "area": "NLD-PIP", + "text": "Offer a bounded, cancellable stream between host and pipeline by default (the original default is effectively unbounded) and a time/memory budget per pipeline, as an opt-in deviation. DEV: DEV-PIPE-110.", + "evidence": "src/System.Management.Automation/utils/ObjectStream.cs::ObjectStream@L395-L1517.", + "dev": "DEV-PIPE-110", + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 258 + }, + { + "id": "LSN-RTPIPE-07", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Create the policy decision point at command-processor construction and before parameter binding: command origin, language mode and session-state visibility are all known there.", + "evidence": "src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs::LocalPipeline.CreatePipelineProcessor@L817-L919.", + "dev": null, + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 259 + }, + { + "id": "LSN-RTPIPE-08", + "verdict": "AVOID", + "area": "NLD-PIP", + "text": "Do not reproduce the \"pulse pipeline\" (an internal empty pipeline used to run pending engine events); schedule event processing on the runspace's own loop. DEV: DEV-PIPE-111.", + "evidence": "src/System.Management.Automation/engine/hostifaces/ConnectionBase.cs::RunspaceBase.Pulse@L1094-L1150.", + "dev": "DEV-PIPE-111", + "doc": "RT-PIPELINE-001", + "file": "03-runtime/runspace-and-pipeline.md", + "line": 260 + }, { "id": "LSN-TOK-01", "verdict": "REPLICATE", diff --git a/docs-out/data/subject-keys.json b/docs-out/data/subject-keys.json index 8386b4e..ebf4638 100644 --- a/docs-out/data/subject-keys.json +++ b/docs-out/data/subject-keys.json @@ -16,7 +16,15 @@ "ast.visit.pre_order_post_hook": "", "ast.visit.v2_nodes": "", "binder.count": "", + "binding.name_match": "", + "binding.parameter_sets.max": "", + "binding.pipeline.pass_order": "", + "binding.positional.attempt_order": "", + "binding.value.order": "", "closure.captures": "", + "cmdlet.common_parameters.count": "", + "cmdlet.shouldprocess.order": "", + "cmdlet.write_permission": "", "compiler.foreach.enumeration": "", "compiler.log_segment": "", "compiler.loop.nonlocal_break": "", @@ -29,14 +37,36 @@ "convert.constrained_gate": "", "convert.double_format": "", "convert.string_to_number": "", + "debugger.prompt_commands": "", + "debugger.resume_actions": "", "destructuring.shape": "", + "discovery.command_types.workflow_member": "", + "discovery.get_prefix_retry": "", + "discovery.native.arg_passing_default": "", + "discovery.native.exit_code_error_requires_preference": "", + "discovery.native.process_start_phase": "", + "discovery.order": "", "enumeration.dictionary": "", "enumeration.string": "", "erroraction.default": "", "erroraction.throw_statement": "", + "errors.dollar_error_max": "", + "errors.error_category.count": "", + "errors.question_mark": "", + "ets.adapter_lookup": "", + "ets.member_order": "", + "ets.typetable.merge_direction": "", + "events.action_throttle": "", + "events.reserved_prefix": "", + "format.fallback_shape": "", + "format.table_cache_window_ms": "", "hashtable.add.comparer": "", "hashtable.literal.duplicate": "", + "help.provider_order": "", "interpreter.max_parameters": "", + "jobs.background.child_args": "", + "jobs.foreach_parallel.default_throttle": "", + "jobs.job_states.count": "", "lifecycle.cmdletbinding_processor": "", "lifecycle.exit_script_file": "", "lifecycle.flowcontrol_across_commands": "", @@ -46,6 +76,9 @@ "lifecycle.script_parse_path": "", "lifecycle.script_processor": "", "loopflow.empty_label": "", + "modules.analysis_cache_flush_delay_ms": "", + "modules.extension_order": "", + "modules.version_folder_rule": "", "operator.precedence.levels": "", "output.unroll_depth": "", "parser.array_type.max_dimensions": "", @@ -65,7 +98,17 @@ "parser.statement.terminator_after_block": "", "parser.switch.flags_matching": "", "parser.using_assembly.resolution_time": "", + "pipeline.concurrency.nested_must_be_sync": "", + "pipeline.first_error": "", + "pipeline.pipe.add_order": "", + "pipeline.pipe.default_out_buffer_count": "", + "pipeline.pipe.retrieve_order": "", + "pipeline.start.binding_before_begin": "", + "pipeline.state.pipeline_state_members": "", + "pipeline.thread.stack_size_bytes": "", "pipelineresult.bool": "", + "provider.hierarchy": "", + "provider.instance_per_operation": "", "repeat.restricted_cap": "", "restricted.default_variables": "", "scope.allscope.copy": "", diff --git a/docs-out/tools/__pycache__/consist.cpython-313.pyc b/docs-out/tools/__pycache__/consist.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c874359da2f50f190b891020647b4a0210a9fd90 GIT binary patch literal 30799 zcmc(|dsJI@o+o&(B#@8<2*evdE(UC5TVNis4R&l}UdAzC>mnOl*jRwT*dXu~_@PjV zlioUkxVjB-s*I~L6SB#6$W*$Ebk*r1r*k@Fr?!Xc(>rq&QHD}6sow3{JtzOnNn&qz zCDZ53e!jn}D+waUshT;vKbzlu{ob$d=ll5m^e{Qu$ie;Pk3YTq{!xzmCwfp_QuyZK z4>cV3eNN;gPUJ<+GrYvJt47kWt5(vot4`9fYl4)(u8C42uG(kx#|@G};5xOAbT#Sd z)sY#!SMLxqOTCX?9l6na^-k1@2~VMPK#w>>tO>_}+~#Tyhj| zZtt*npvOJ9Tj;su>A7O>8FUW}`F0CPeAGVbk?3Ce(kBh$)sTA>Z`F6cQTOPWZ?{p1 zxNGjgfnJoyE_wPQQIXgcJPhFs^)o>5=vW#6z@{lw~l!1NBa zYdX;;inWfj6gj52rPqh0D`g`C-q^UKH%B~bVs&k;?T*^|=DPOgwpJ0bOO0(~qpVyt zf^W1Jg+fz%d(js@VVB^!F))gL>h%ca!mw9ptZhDVK)9$hpwd`^-F{sf81-Bf&?&-2 z)+QIx9qc9RHD9UGG3I40S|g0y9KAH`t&Fw+b-b@EW?UTH*p-6IZf7~3ACK+R@o1Nh z$F#uLJ2xCFTwEl^igtc{UDt*Kv7Q}gl^Ayq4jwX|aC`g5-2I*!;Up`$P$`tu+xP6L zu%SZj9^a^^w?;TI;2pam+}OXjYi|{Xts1hlywdn7&4N}NA2RiN`rKoKqrC$?qoE{6 z!^yTTM_XHah%faU)Zq})(J=Tbf%T%_pmyUST5Y66c%cN=T#_F5J>iuUw;V?w3-R}` z9d8Rc0hGW+@0oLi*IxCRuG=`5PJLGAvz0sc0>_OZw~=(O)~T^_r*%F}8Rx^@v8^08 zT<@ZMm^*-cdXa8R3dcFQlJMJJ%zY>5OU?E0SFE5;-APSL0mqq8Dxb!yb819TsqLIg zrCgj-8&iUV9Oq4NCOAytBx5e3MhzFqk4liw#n#mv6Cr%-Nd5Y|j!bn`y~ce%LEu^r zDJ5+A6Xn{EvMS~6D>Z}dq$_Mrhu8kWA&V(QooAIH8%+{Cd-tpylijk4{~Ex9Sc1-L$_xzvx~%7-&3m zFYW%(faUCjX0m)DVJ$6tM)QNbiQ3gv%N_ruf2MdPRS2dEEV=3adxO)1vlYQC+p@*> zFo8?o^hE-fUHE0la`sCfx}1Bl_-G0LcJk4}#03+)kMxP;Czs&uN7ZHU*9~GCkQjX2 z!{hLtG=Lom2(-aDx#$2@ySIpYhi|p%BpOdr8oZER@(facNXdA@%;ZOW7|fg?5p)S@ z@m0cL=Zf&>%Z7)>GbP{IH@WZ5;mN}@<*)SwP1|JMwukui^-0Le??8Ir$F!j9Mkn$c z2eJFlXbL$f<_0HfM6Fs6FYn|<9Zf9vHICcEc{NT>)H-<`7q2A%xWaj3>WY|B+Gxu- zFiAJMZJd~hX&7E8FiCamrdT0OR*4*G!2+&a!H?BdAy94oBoXW)O5Kzw~ zJKl(#j;qoX;p${po3Yiev)k=Pe}a)NpVD~w^uNJ{c%wh-LW!%^e$H+8cb(s5tLgOZ zEGgY-EBchG^eMH)r!-(}`j8QXKpG9tOCoil1mDQuz-UM#c|r-m13i~Q3H{RW*a$w1 zN&_Rdgph93b7M55#eBvr9SUg%F;jB*Jjie!hPL?-sYJH@)=y z3(rqoU(s(3=(nz>q~95s9GFR(Ef1y?PiR+7DO1~SSFfg7CyuUW<;|4Oh*MWqvUUWr zcFgG>WZ9=QYZ+Phj=geB-n@4~6U;a;#sAitF;%`=y#4JnZ=9Js_2#*i;-><|PX&wj zO&z^=efs*$)w}+?t)J_;?9$H+Tt?PR-E6`ArkP_?&&k$e*<8G8N}bA>sF8JXZ#m)K=@fv0=i)EH)TY(osq(vM%Cm39VlxUA%a) zbmw9CAxz^?Lf@df-{()d=(=#R(|dmB#doyQE~FCD0vCpO(Z)+7&u5%<8=_@15E`tC_9;ndaxZ_jCc%Q@84AwnyFvO*UC)W6k6*Jy>A3*EQC# zfp&hEoyHq3oxRUie#d6FAHsk;Y@;w`b&Ysj*Q_+>ShF5Q2G8IRrUs8eeJ zcn|>`Zq%%$mg73a2;?{hI1p(!ltj;e9YWmvBJmlK-S!97At{<)rDk1OLBK$` zr0pn+-!$gE;vK&373vxV)=7SScuA3t(046XjZhLXDxM*achql;E+&3!@4z*W)Q{JK zXQD?-`Y-fkKJykr5CmhFxrlQYF3PdqP^T=Xyxn?d}a}d-{CR7Q9yR z<*i${80C@RM0nh@AZ%^vq(tr=Ql(@UQFd>u32N z8H1L>iP}%B8T7$8ZG5c`FB@lD=4*f6^j_0K?N5&fiVn&7hnFo!Rf?@1)2(_Rag{^rI7XtEuVt`ltJ6+;=YnaIQ;Q-d?>>uu%X0j)h#g zs7}tWU$!)?TC*mOvvoSeUsswt)6vjc-{7#<)wZ^^HrLgjVEn5;_iGGhn7^zE>4?D% z=}y*);@5+bkm0&CJnFF}1ImL959tPn(XP5-w0jg^O0juag&qU47*&F4Mk_*mw?trAdWJmKcO(f^cmv_cks}K7 zux#u)={!P#qv3py;C%&Jx3%!~n?JnyZpORaKk0v`|0jd*41Q20*PaQMb_8u_m$#k^ z8eJc6JvUP@(LT9vs`llYz}9mCqw7}PYI5pS+O5W0brWf8$!ULfdV*gwTc*-x&IHWG zE9RX6^Uk1o_XPhhIzv+SqjV0?rem^WswbGdc|!Bs)QpL-ne=~?&7w@z&F(<(74x2e zdCz?Mvbk#2oII5_+4yqp=UcgyjDHq5L-KFZGA3%*%xQAEO*WU#)&9y{`ni@%&-|Ss zSx&M2wfN57g}!A&!zUTp6IW%!mYGxV)^tg-v3U0Cvd*?@GJp9+IcMtR*%<#wpIX03 z*MlaAIAS^(9ZtcDaKQ{3jO9;pE?PF3i!5p}OtK>mumKYtVTj|Q>L<}XSKm~q&sz0k;NBubWowAEiZ-X z={L7=!G`4U_(${#mV3oPxEme^YR%MkbJrl{NN@a)$wTQ;N3oiF^s4q85QEX1Aey|1 zAbUHa-_<)c9LB<%;jwVTX#lcIg78LLu3$U#Gl_KsZsb*$O^pdbjhV=zTF(JZ3k3Rv zV7x|e67U#7Yv!CrHH?^0^J-YMQw^LL$tNr%GDY)2>dBNkgg-!43`uIeI&adD*+t}v zIj6abCz1GN0|Y*V+&4HeRn6BL5htr}ye6k9(la!oUqE>h#RP`o5?v4qDNeLF^{#UD zDW)czn5N^}%@9VV>$s=Fvm2xDe+iGeXbM(mVr))g2Kpq^MFf^P61Sso)Q(VJNz(?C2(i7#EBj0(NwqM8-(x) zecaR9ty~`vKTss=i$08w=)EnsHI%5x%l-PWpjoO#Cel&z7=p>GJJwM5jJ@teZF7sr z$f*EmAuwua0!+B<=^5?1;<*`08j(B@bppwJmG(86v$T&Aq-hpPfE0Gj!{E=?;3>86 z!&l%P;xF)pr~}DVFwOy2hIkj=D=xlxQ{>tse9zMnXzE7$p`~bGGUwzm0(lyN?HZD{ z=#akJYwAb~3kaW9#znt|i34?hsup;^U8Or64Px7g(+&0ODuC(6u?p@z^kvlIXLmIo^lRQvvy!dfPmn^m~9sRrOxckyaGvH6sGWb@h7N5Uu)Y!esS`h@bZQlKfhFIg$_p8fcWU-tCBR`GiE z53A+k>V=A6{-Krpi@&H`ZoDXWxa7_Y^7#vaj;>(i#bDjV<@}3?@Fk<}|5L5DR?kaC zNX(zPZd{^g8c#sQA#KCAPlwseLQ6Xav~F``6ODtq)X({s8%4a7WH8$H^f7X zK8Tv|T_GK!`a%ZD;~O5l=IND&5Znhsf;No0F#I*gjydVO2tZcj&B5$L50Vc9*k*2dZFIJFULV-Hf1%((=E2DZ z$i367_j0Cl?&eJ#{luF2YfH{&8eMYkT1L*j``#)1IK6bDZY?`+IqzBdig)G8XyD4IJa%*Wia(h5Y%u%TiDSRD=1=Vh z<2H45+AyVEvt-Iy_PMJ++rE(T)7=ZwA}?1QU94NuETt_rJ+PekJe9K+JhE^J#)+~& z7w_+#>-)eOICy^P^nW`4%k%%x6+C!8SbbjJe?h*`FJHPM_Yca0*W|s|mvuKju~;X{ zWnJD(8ob{>+RA0*tXOvhtUDfZ8uS(ATQOg|Y^nZDcFt!Bc=fLjQ#ec2m;aPr>LZ@& zf90~8t-AlMJiqyX;qQvmkMo9KT7~BQ`d{wj;rAd!#F0>qrrg7iu~s}ufEQN+5&^SO zv7DF0`FxB%j3ZK14Xs-fe+WGau(|VN~<^r_1p#rnX^$Xp$hr|FHWPyAESX(^N80*qAAki zpywN%8k161>&04wV040Lj+DAK+BzMX-^9`pFVNDV^hOiklC9i{dI%_SuQ)YV)GCBp z>Ax8?IU<5hH9pD`Vep{UMQ*GwP=P4ej2X9jW!%PN1l5wLZN!+?Nmqm&bS9}^yhbPQ zD2SFoy;D;_T}Hx;jw_~riPUd!=-=d+5Cjx9IIy@8IV5oS4a|fFa0|%E*W(`Xg!pC!Q_+A29^(5$I;vF+3`*#MV&5mVP$I|4 zBhnq$cTNrcTli5&Js+eE-TbjhDfAM?a~* zW1cj>Dgvz-N^aGznv$XNN=ct8pSk|4lpPbA)uQchAA952o5xp*sscq-^J1{5dZp-a zpy=?TcDblNXleMksD9?^z3l1iALIs#>L(IpO9P>WvD-B>X@7cnW^DG<>o@M-m`|8@ zzh_w4Qy17%C)anZ>^U3Ub5_1EAYZ*EU%wH!dUNIK^MR|+1K3FXQO*Cuab38qWoF;& zc%@^ee|Bu{)EhS*WbT>bS2Hs2HBC3&ZNAqw-8NeXjJ1+c8ptROX5b^!go5yI+xx5B zy{p@I&DFkfX{!G9?EBe2%Dvn6NY7=JLl2r-@>vRJGT*9GWtfUWf5ns^FhK+zG;Nu+ z2Tf(aHfOA*=e<@n%g;3h@~c+zs{{Gf!TbZk^n;Un0L0YXnWA9o)(PEecK$v8wEqX= zlT8p;qRg&WT(g^hmj3hHzsg-G|LLYc!PAT8K*5Qn-sSewEA1VD_KpXcXHk_{FqBjB zu66FzkF(|s^T`YD1#!`^l(Ce!)GcS7TDCYsmduAJ{W<=jmdh#n+yn{fqYMH^_CK$@ zpEcL`Gv7khPjAj&k!w%OXU@r|U9ziJuJSDF`qsgrFTW_}(i+3!(Bj+@11UGLnLc00gik)a?Sg?) z$Ox}%$OEgc-cV{pchJQQ*g|^W@R-!&@rBHa*-Y5tC56~mVn=$rV1+ld4q?%>WC-gO zhOk6FjE*pX6&IxgO`2l>7f?x3c;;dld&bT~1*uoRMXzcKD#c;6s0tVhocTr7=8r2~t9`xAhy3o@lFkrlFqhvrjiT znj4{-Bx_vz(b~FaPP8@QgVc>gzDjAXw>uh|n#Fcz=j%7swIO_M6J_L2g&A*ML+#P# z6V2^p1?*3cs5$M8wRP<*ue8R7+IFX-!QN8a?r82{S=QFoHJog(ZLMptpK>;csvWR2 zN9Eq4$k7^G6xptKSYT$-f?_6JqhPd@^@04x8ewR_=OgoN=?zLHrACM$5rXtT<0X^% zhWKrt9;6T@umq0K4WW0S4+zpnc?}8cO_&+2^5?J5v3CpxTO?bC3AXT+rbh58R_APKXl=&_6f=gb34W_0BaAf+ zj|G)xGFpZl%{vM#DczyW(ij+2ARxnz38}v*RSH~$ z5e&UOlK~kO7<7kJxfY1VX54c-0=X3{x%&dS`+~XEK}*eK0$`xYe5YozX3D!_Dh`;6 z=eE7G^Mmvcv)|ADaO?Y9m(qgO&-{ZgSnU900)m{ZSxZeLNz_W})l8%Qsj4 z=C#>NbC(y=7PNBK!DY+SVc_(N7GN$t6B0Lb%ALKFd#8ZfgtzMEk^@D1gQmU6W2*J7 z%Gs-L?wu`}D_uzcFzfxS#pGahtGvHWPCvP9J{8FqrDiF>`YUKETrt@KCfi(L&{X=f z!ui2KMcrcEqC>7Y7Bn4Om#VzeA*YMW=JuF`Ql~0z*Q}R zkG^+ozGlJk3+-alpUvkdE=K)FO|tz=awz5C?MQ~R{K%oMBP-?v|+ME zEgjWQE;;kSbY@*CzOin%>prh0v{(bQC>yu_W#t=L^Nk<)7Aybe#=_+#v+V4U+t12p zyXAvD%evlmXi*X{T6!>8A`Z){0>{3<+b0RUpbsaC!rULBAB~7;qM|XCn5b8Xt2FB@ z*0zN$)-+LzHLo_-N-es~dv#?1GEv|aZ=(}L2C%7BNqvL>pjKo>R>m`X)I@-y{QyOY zPfSbiOthml0IlGEJzXXM0xgC1GeWkM)+65;LzY{hdnS=UOpc_=+2Bxm12mCw#5iIS zU{$zZ)hlXo2({3I=*EfR)bHWcknz4p#a*H~VtWTD62C5DvXixe+K&)&90`NK_~nfN zDlwJ8K8Rr+Q!|U2rkKW{blPLfkgkRlP)fCLd%f1WI$H_3O-cj4wP8M4Uy^_0c)SpJ*%Gg`bOX-S70x>c=TNN7LP#;RH zJqaMt5z>bdfM3(u?N5cRXsjWr^d@DukGy{ik2JE_;neF-AS)w5dY#_s`o;za8M%}G z0?+=<8~L?@rzwO`LX^-JGAOJZ1T^{)m_84<2LV$~QVyrcV}xsiG(@BiOr8O(&Pi9u zYl0WjlVk(BKt{7{28mIn8Tx#Tyl2QOBd;1Bv^nnXkOp>Vw2Y{F8--q_U>_H@9eKJG zo>Te|Ar|oGI|z>9k26@hRVN!=Yo^Sp-k>S}R{g3ut zp4HSWId`|5W0zCyvcdkF)QSl$nfzA2Ts`HS5vN_VPX#hd=89HKWe^Fhn$zx_m^?9K zePG`Fc>-ste3Zz6)Lnl0vYfkX&heAe@0^ya8WyFX{rH2FXNYX3&{lz|iYe*#;T6-C zfN9Ijji9My?%1zPFnCfD%bu;czv+Q_I}%Ji!GD&>8H_Jod*Pa#A_VmXv!{dlUGtis zC%%_B|DE8ThNZ0Fp0-s(GD&UZ%&KL>Q=e=poH#!3lufYUu9?eK@4oyZKWh1;OKqsq zty69$qj3q`KD4lv9E>&ZLQ^YFWr?vS{M~w%8g{Wv84*vJp~iwLK(D40-OMwFd`23g zI|2dxoubBzUPVK70_qWl|Hw&$HIr(tji;oD))Fh(k5G~bQ!W~#%-A}H z*re77j0Qx;@UbR>JxtLN3l>p&U~PiADr3d@mZk8jA5A?qrUlTxqNNSu@<^M|n>4UF z>7X@HMke|l0*+{WqCOF}#=4QMVXKedbU~S(HKjX?{D$zoz}Ntr28N5nW8P6V)tTH9 zJP0!n5!=Cagv~?zCTw@~KyU;BvA~qqY})q#CUd0~Ly^c58ywGr~aY)KlT2yH;^Gt0jg(Aotn|k_6IDb zE0(H&rE31v1IxbO5!BrIZspw7AMb@Od)GquV$S=63m2AZn-^BNxk&*G9@k=&EKXBfLo0=MIy`Xdt z9@>pPKDNVnb>u_|Y;fqK{lsB*YddT^+=G%vTq^F^!l%LTzmWd_KZwO%7%CHUQ9!4H z6IjxdkrN;pEY;H$cuNamJY8UPfx$AAuPheOB|l-aS3K=y%yyBOZD3CT@$o=wMB*1& zlz|w`if5 z4LHsuFdjUzsJg12WMK8gSeC3|Okdbw*zbVF8G@_FR zvEED?+ja76eYlqTmT7E2`fCPjfT}UNXs|H%Kc+U~7ICX60Em1;Z#pW~Jfl~!Ky3$< z6t6q3(9m%vtF7Ypq2_Am;k_EV;|f)P8TtaEIa4Cx)7FKLuxK&ix2_9c7e9QwJZsVc z3c+i{%!9}EL?P3i7deyBE5*(f_}i5d;BH4{#0o=z?Sk4vPP1BvUQhzaqc5YJCI@j@ z8=P38w!(%Oe}Wk1cr!X?t;wKv^QOXnfndc3=QNc?7;1)ZBE7zoL-J+Uu!!$fLlz~Et+mv~Z zkbLXEs3F554GTmXRXE8YdL#&h2{>8BpFTvp$|Rqi6v4IZ^pKD~{0?>rk{W>yH6iW7sO0giR~A5tARq&8r5h}>j?fNwwcoI8|bIw z14ahjJs#Nh)=@HvunQTMe9R7cfcyI&9`I+>2m?b1Gv*uUMq~*`h#(2gkrUFNB7)x} zl(0lExqX6PPbbrOd^W$fvGcqY9^Z*}ZI78pU$DLA2|h8sPe5wdoudH1r0|-dk&wRU5{-}EP@ z=e&%%2NlH+QFdmi6iQUu4(AV)*McM(>_In82%$7UNFE_v13sd}k_xHVi@czMfb#sGc{gQPsXnq=^w3OV}ih?Oy zZ#DfUIemg(HKa~$pH2!G@>UE50YkxT!2?4P88%!0F~xGLX)VcgCwnsc&ZfytulCBD z_5_p4ZyhCR-1Qf(zx3P-&rNl&=yL=5+_kjKd%3UV!stfxqx}!kil7NPTDREz(e8<+ zscX|`XZhJ(Z=|dURRMuak%a1nZ2{qFIpa{!e3*6y7JrtgO~_nJ-*m6^mD1Vbxvcrk zP`4cWD0%6ce8wf8zbJQm<$~emv74W9+%uXs4KAlO=g5NE>LPP$Yd5>^X7`ua{Ux%t zwqDju=>Hj_xzsh7KmFi}6V4N}3WQ``yv`_TaXf73+b3^#C@x&J^7}Fk3j6@#d~yXKZ4crd{vW&GSEQoZCNt zc(H16yPVa$Y&jlLEWu=bHPsqSE%;2MP206rxZ_>TPxSBTe`0>e{Im9js$j{Hr2}%; zr9k1pN})GU=nWQ*tQ1}g6v9aB*srocS*TRBrStn2suxc$9$RXaU0w3Ii}Duta!&VZPX5f*`-UHBAMNAP z3m{2O+41Ng)A;56dC&dExoZo>3)zeMCGFzaQnS48^s?^EI#u8AAGPqDITQNXgv{0S zjNAIpw7C1Pk9Kl71$ddZYqfCaRO72B@H%bRzkZRzWgXG@h~oT0TdO&h!7WwPrk>iN z``;65P8~G-M_nTPK!*8Lkv34slfNV7RJA^^pNHRrSP=(|qDW)Ni~_P4P!Gkaj0mJ7 z_@cOYn{MM-bh#*1=IA^URrCn~_cx~$hntM53roT&y8j87F`}Au#1S2n(lEL)6yjUu z@C#avH#oJ9DyoHSqY&tyPC&@q5EPHFNX!2E+bF-Gas`n60JAgo;FHR!|D(%!ft53U zi6X-5xQfCE7meD2 zkN}}J?m^SZmQjE#d|;41keRW2ke&Zm%Fb%@8>3nIpSpOM#HM>Xd&kT7*z6?HgqT#3 zn0DGbcb(tGB&HCFdb+)RohY5ncJc5A^y=RNVP|Kr8rxRZ`D}^n!gnAR?fec!VemJ^ z?+)>0k`aT*mVzx=I4XjM{6?#1hKs;^a|)<&&>b1FU{6>`o_IV! zu-?mmC12j`Tn0hMcEEv=z?rKnXRZa#TnnDL@gVIcNUAyYPSa%5R4o|_L-c<9fq5&T z13&*e?B8Os5B)+1gAT{pz>#w+N4f$>x`IdC57N33H#6(rndvh#r|zCx$t(_J7SHxP z$lS?nb=fHwnmYfQdAwKj&UzeamChbg80la6_p|bd=Yai zAx06G5MGCp)UeTOB(=mCrC4O`h=#+;mKc>)M-399>lbz?L6!<50)0F+S@d`Sufb`E zC&7NqZXbxN5I)YDi0%1diXrmU2$Zhcip-qiI9f7Gt$_n&+vwuioIW826Jiig zu)#%lnm7$=`>Az-?Dm_>T(BWAJVtlSsP$*k%)ftP{XuayI%kp#GIqtOWm9A{Zas3s zZY?Iy)wO(y)QP)h4ifT+L^4`CT7^m05xWvcRhp2!<%ek%%?jomJJq}B6(bKy(wNb? zJNiw%Q|UbD%74R1FhWD1lvDG~a_UHXfwTJNka0&~L{6%Vz<6QQ`K5$_^e2UmD-cte zj9BN8vbgVt$5wQV=50`q=c6|^IeCoI z%}yQsElv&mt+0Wn6&jA(IWj_kA0sMqK~a&nO96z6l4DR}Y-j2$uywjz7dp?M-`RP- zvjnu**6AZ^Y*UowI=}wX=;%mIS()EJH$J-44?&zUI$RUt3#H#9QJg8j^yk%L5?Ctcj6X)=P37?qJ5RMZ{>ceIwNEM_}I>*pGaw(MR z^IRRnDPW%Z0XN1>JBp(ZM+Q70{blUt@ZxX> zb8K`!&NEiewF(yuGCG!zBF5Sg;_=@V;;|pBB*fc7{IencJL~IurqI1qXYiq@-_#@oe%s4}lu&H8Pyoy(_2~*G-Zg6}U#H@QY5R2a8>#3`RX6Fkzfl6Ds0&6+_XlbQ0n zH52uabKH4;@_Bh{<4S5%Ahn4^AJ*)9HPbaSJ$Db!mItiG6ZM~1v+;f{J>%Yv=^ZQS z+XCs^-l}+e-y8ehKKRB#oC2wTPyh4O_fi-6;GUz4^})jCVES?FbB~HZ`-khi5C_)dsD*=k-DBK04Fv>hy`JCRIFRd$(}T{p0O(IrCc@oaA}1H0a^A8sY1Dv(hX%-B1B_2Z1{rS6G_)zl2gZ#KJ0 zo@H@sm;@$L9J&0I>^LKzg3RUY24iF#y>{p3Y5_e|l(9{10b8I=kfi_ZNbdBKDuHoVge{13Q!6;2DoRFt{=x1qLM9cP%(@{a0tM zZ_N0*oONTxa`RCtmvMyuynsvD{%8+bpp^c3-Tm!zrx&#IV++lT{flSi)|;|_T)z1o zRO2~*goJl{uWBA~T!N%oC(V!qjK+1C8@NTm+$V7JWSfs~dyXkMNL{P67ilUsn)~Ul z1|H5Hr7070_NbJEq0hVqngf76m;nvdaP!iS#4+ncAN zESP)K63JZDYGDj(&PbIra_pJA7_>YgjB2q!c4!sdDHXy&HQV4~_@2nFYPI~p2yqWL zjml<#N&h3`$I{hg9U1Cf^a_-#2XaP9$>E5v=mAh55*>+g!fRGgzV}y?-ft)&`P&bsoYv?e1^QeSv4YNg3<3YNG z^0FGJpp|n(?vy;pM-2q^>5nKkSw5wxhoq@_qs&K8<|OBn(k8&O!9}@OWryUq7|RjP zU(FwNKdRJSw0Mm%$I8XFY6FcJrx8|FqR`0<_8gcksq%5#`3Wq`& zvnp|za=iMngZ8r7L3^gJ+elD>M7B|darwsouCY!k0_MaVbrlhF*)fOOct_+#59AX# z7o@}CxOp)(jMQDsk9ixZx47vk8a!~&aBsG4X>HuVph-vYjd4i+BNsZyaWZ`8&Ydw- zb{v#-ysmK^baNahhmALhnw`E~e`q@0 zY&%IP0jB`dhWq~EL7bK$eT)+NlR%Y~WA{G&uQ=4+|9=$<&Gk@y_d^CP-834T)J^k3 z_f6sA#vguAevRg0sc;cLGy;wHo=x&83PZ_mT)bsw|#a5jz87=kj@IQK5N-BK1n}{GdEEEPIFL z^CLm=s_c+{)$9vL4jX1v?&YO-sHiRQ{E4g>HTYE;WgPAHXT%!|Y!|d`6FY!TVRiE; z2?u%YgUyHP_=zWX7kSq?c63jT#D(-i?wsO+s{hr|!gPhwb zU$_{wxN+n*j1Vjts`esa5mqck0ZY+a;#}IBrV zlL`cz4yN_@Ow%UiOz#JlLeh6^2j^xL=uDCw;9PO$Go7k8`r6Ugo9;Kge&YU#cQfR& zhTx{gV0P1kU??VQk!JpPagXJzTgVS)H#|sg zWYNFpxN~Om%u4d+K=S4pcQ9F)wa#vnlZyj}?JI_z0Yi8T!~;VGrP;)PrsI;XX>Zl8 zC0nQJ|E!bJNSctQ3Z|@6bu+rxlkO+IwS6w*o$NQuf;km(cIA8}ou~5Q{`dDU)-Fna zb2zx~8F}xCrFuE}RKRfRWu8*a`<>D9s&%UUZtiR1Y{rky2GX_%jXQ9vo-RQ)?S5CB zxBj?e?)bb%PCB?S`r*y@Z$46H>gUQ##Z;hqi^~yQ@X~mc_}KC))~H+ z9A!3Ru7*#Q@78N8adzoi2)*e*{KLPiu@QAV)sFA%~I34u_GiwW@> zLL@~((0peuCd6%oFtQL-9$ImjgIZ%_d9ER(Ig(bI8Zss=4?-kI(n9+wPE7e)S|<^b z8TPuVZkcMxn6wTfgodRB#rD@j!1U9S$3;ycqRNw~`2ziPV06n#lohO^U$=ln&#Uby zjQ(_=Jp*qv^b=okY!l+0vCCJ&*%#}| ztAbkyqw%ACes+?T@}qrz<#+>o?AZ)zXl0WYCBn|t_eTufm?5EGkw_}SU<#zjKWQaj zH$h0QLEZC^XhQ4+!DjUEWP$8Q)7lO}&v~j+A9(_Kx$r`UgM;p&?q2tyP!?qK-WW0- ze*Xf|4ZeiSC$(Mf8;hWCSQAYd0?Dn zXEQVM32T6xus3x}{nwQ0e3w%DA$fmA9!vbc&>abJ8Q}}*=+^-8Q(DLHI>fs}yd=>; z*r5H_QT3%=h;;>jz7%*Ml-YT&RldIe{{Fk)nY|j!F8QAJj$zU;mG-h}&Vv=tU{W5Z z(kIL~lFX0apyjORG{4j7Upn^Uv58aPeTH>I$k2sTI&cI^myK_QbN% zLqpVi3S#;V-EBuhhD<%fuuJJ7t5lx?y=c)1>D}EvGPVfmJ>F{r_+d*09sOsDmrfqT zNg~~yB9Ec8_vj9DJFK5zI)f^Dd>S6?Qn5dq%vBH4D<^qu*hk07JiTLs6^uO3y|S8yQr4GId6M*AHM!qoLtm{GR3^sS+{ppYt`rhZP#nf37p~ zx`$aDZ~hHu`VE)F{-^zhGyZ3;@E^mLrSv;n5}nO<@44ycX0Oaw2DA4C_4{w>Ruc^` z^}o>n(%=h&Q&ls0!Nj6l+SPER@fXGc3$lWGfx;PI8hCMFS~uk&V5Jxr&R(B41~U%? zEeC^%Pv6pp^aZzcj}jC3onKgZzVHi!khR^^2 literal 0 HcmV?d00001 diff --git a/docs-out/tools/annotations/T3-docs.json b/docs-out/tools/annotations/T3-docs.json new file mode 100644 index 0000000..fa0f6e4 --- /dev/null +++ b/docs-out/tools/annotations/T3-docs.json @@ -0,0 +1,268 @@ +{ + "T31": { + "doc": "03-runtime/runspace-and-pipeline.md", + "pester": [ + "PipelineBehaviour", + "PipelineStoppedToken", + "PipelineChainOperator", + "RunspaceCmdlets", + "engine/Remoting/RunspacePool", + "TaskBasedAsyncPowerShellAPI", + "NoRunspaceAffinity", + "engine/Api/PSCommand", + "engine/Api/BasicEngine", + "engine/Api/InitialSessionState" + ], + "xunit": [ + "test_Runspace", + "test_PowerShellAPI" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/hostifaces/", + "src/System.Management.Automation/engine/pipeline.cs", + "src/System.Management.Automation/engine/Pipe.cs" + ] + }, + "T32": { + "doc": "03-runtime/command-discovery-and-invocation.md", + "pester": [ + "engine/Basic/CommandDiscovery", + "NativeExecution/", + "engine/Basic/NativeCommand", + "Microsoft.PowerShell.Core/Get-Command", + "Microsoft.PowerShell.Utility/Get-Command", + "Scripting/Requires" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/CommandDiscovery.cs", + "src/System.Management.Automation/engine/CommandSearcher.cs", + "src/System.Management.Automation/engine/CommandPathSearch.cs", + "src/System.Management.Automation/engine/NativeCommand", + "src/System.Management.Automation/engine/CommandInfo.cs", + "src/System.Management.Automation/engine/ScriptCommandProcessor.cs" + ] + }, + "T33B": { + "doc": "03-runtime/parameter-binding.md", + "pester": [ + "engine/ParameterBinding/", + "Parser/ParameterBinding", + "Scripting/ParameterBinding", + "Scripting/CommonParameters", + "Dynamicparameters" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/CmdletParameterBinderController.cs", + "src/System.Management.Automation/engine/ParameterBinderBase.cs", + "src/System.Management.Automation/engine/ParameterBinderController.cs", + "src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs" + ] + }, + "T33C": { + "doc": "03-runtime/cmdlet-infrastructure.md", + "pester": [ + "Scripting/CmdletBinding", + "Scripting/ShouldProcess", + "engine/Basic/DefaultCommands", + "CommonParameters", + "Scripting/AdvancedFunction", + "Scripting/Functions", + "Scripting/Dynamicparameters" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/cmdlet.cs", + "src/System.Management.Automation/engine/CommandBase.cs", + "src/System.Management.Automation/engine/MshCommandRuntime.cs", + "src/System.Management.Automation/engine/MshCmdlet.cs" + ] + }, + "T34": { + "doc": "03-runtime/provider-model.md", + "pester": [ + "Management/FileSystem.Tests", + "Management/FileSystemProviderExtended", + "Management/Registry.Tests", + "Management/Alias.Tests", + "Management/FunctionProvider", + "Management/Variable.Tests", + "Management/PSDrive.Tests", + "Management/New-PSDrive", + "Management/Get-PSDrive", + "Management/Get-PSProvider", + "Security/CertificateProvider", + "ImportProvider", + "Management/Resolve-Path", + "Management/Set-Location", + "Management/Push-Location", + "Environment-Variables" + ], + "xunit": [ + "test_FileSystemProvider" + ], + "sec_prefix": [ + "src/System.Management.Automation/namespaces/", + "src/System.Management.Automation/engine/SessionStateProviderAPIs.cs", + "src/System.Management.Automation/engine/SessionStateDriveAPIs.cs", + "src/System.Management.Automation/engine/DataStoreAdapter" + ] + }, + "T35": { + "doc": "03-runtime/extended-type-system.md", + "pester": [ + "engine/ETS/", + "Scripting/Adapter.Tests", + "Utility/Update-TypeData", + "Utility/Remove-TypeData", + "Utility/Add-Member", + "Utility/Get-Member", + "HashtableToPSCustomObjectConversion" + ], + "xunit": [ + "test_PSObject" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/MshObject.cs", + "src/System.Management.Automation/engine/MshMemberInfo.cs", + "src/System.Management.Automation/engine/CoreAdapter.cs", + "src/System.Management.Automation/engine/TypeTable.cs", + "src/System.Management.Automation/engine/ExtraAdapter.cs", + "src/System.Management.Automation/engine/ThirdPartyAdapter.cs" + ] + }, + "T36F": { + "doc": "03-runtime/formatting-and-output.md", + "pester": [ + "Core/Out-Default", + "Core/Out-Host", + "Utility/Out-String", + "Utility/Format-Custom", + "Utility/Format-Wide", + "Utility/Out-File", + "Utility/Format-Table", + "Utility/Format-List", + "engine/Formatting/" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/FormatAndOutput/" + ] + }, + "T36H": { + "doc": "03-runtime/help-system.md", + "pester": [ + "engine/Help/", + "Scripting/ScriptHelp" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/help/" + ] + }, + "T37": { + "doc": "03-runtime/modules.md", + "pester": [ + "Core/ModuleManifest", + "Core/Get-Module", + "Core/Remove-Module", + "Core/Import-Module", + "engine/Module/" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/Modules/" + ] + }, + "T38": { + "doc": "03-runtime/errors-and-diagnostics.md", + "pester": [ + "Host/Start-Transcript", + "Utility/Write-Error", + "Utility/Get-Error", + "Core/Get-Error", + "engine/Basic/NativeCommandErrorHandling", + "Formatting/ErrorView", + "Classes.Exceptions", + "Scripting/Trap", + "Scripting/CommandErrorHandling", + "Scripting/ErrorHistoryId", + "Scripting/TryCatch", + "Scripting/CleanBlockErrorHandling", + "Scripting/ErrorPosition", + "Scripting/OutErrorVariable" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/ErrorPackage.cs", + "src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs", + "src/System.Management.Automation/engine/QuestionMarkVariable.cs", + "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs" + ] + }, + "T39J": { + "doc": "03-runtime/jobs-and-concurrency.md", + "pester": [ + "Core/Job.Tests", + "Core/ForEach-Object.Tests", + "Modules/ThreadJob/", + "Foreach-Object-Parallel", + "engine/Job/", + "ForeachParallel", + "SwitchParallel" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/remoting/client/Job", + "src/System.Management.Automation/engine/remoting/client/ThrottlingJob.cs", + "src/System.Management.Automation/engine/remoting/commands/JobRepository.cs", + "src/System.Management.Automation/engine/remoting/commands/StartJob.cs", + "src/System.Management.Automation/engine/hostifaces/PSTask.cs", + "src/System.Management.Automation/engine/hostifaces/PowerShellProcessInstance.cs" + ] + }, + "T39D": { + "doc": "03-runtime/debugger.md", + "pester": [ + "ConstrainedLanguageDebugger", + "Set-PSDebug", + "Disable-PSBreakpoint", + "Enable-PSBreakpoint", + "Remove-PSBreakpoint", + "Get-RunspaceDebug", + "RunspaceBreakpointManagement", + "Debug-Runspace", + "Get-PSBreakpoint", + "Wait-Debugger", + "Set-PSBreakpoint", + "SDK/Breakpoint", + "SDK/PSDebugging", + "InvokeCommandRemoteDebug", + "Scripting/Debugging/" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/debugger/" + ] + }, + "T39E": { + "doc": "03-runtime/eventing.md", + "pester": [ + "Register-EngineEvent", + "New-Event", + "Register-ObjectEvent", + "Get-EventSubscriber", + "Remove-Event", + "Get-Event.Tests", + "Utility/Eventing", + "Wait-Event" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/EventManager.cs", + "src/Microsoft.PowerShell.Commands.Utility/commands/utility/RegisterObjectEventCommand.cs", + "src/Microsoft.PowerShell.Commands.Utility/commands/utility/NewEventCommand.cs" + ] + } +} \ No newline at end of file diff --git a/docs-out/tools/gen_T3.py b/docs-out/tools/gen_T3.py new file mode 100644 index 0000000..c49d44a --- /dev/null +++ b/docs-out/tools/gen_T3.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""gen_T3.py - generated regions of the Phase 3 runtime documents (03-runtime/), rule 9. + +Purpose : fills two regions in each document listed in tools/annotations/T3-docs.json: + ... Pester files and xUnit files that match path keywords (data/tests.json) + ... unverified security-seed hits (data/enforcement-points.json) under source prefixes + Also: --count : prints one number for numbers proposals. +Inputs : data/tests.json, data/enforcement-points.json, tools/annotations/T3-docs.json +Outputs : rewrites the regions in the listed documents (nothing else) +Run : python3 docs-out/tools/gen_T3.py (render all) + python3 docs-out/tools/gen_T3.py --count T31:pester_its +Label : GENERATED; test matching is by path keyword (APPROX); seed hits are scanner hits, not verified enforcement points +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys +from collections import Counter + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +CFG = json.load(open(os.path.join(OUT, "tools", "annotations", "T3-docs.json"))) + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def fill(path, tag, rows): + t = open(path, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()" % (tag, tag), re.S) + if not pat.search(t): + print("region missing", tag, path) + return + open(path, "w", encoding="utf-8").write(pat.sub(lambda m: m.group(1) + "\n" + "\n".join(rows) + "\n" + m.group(2), t)) + + +def match(path, keys): + p = path.lower() + return any(k.lower() in p for k in keys) + + +def pester(cfg, tests): + return sorted([x for x in tests["pester"] if match(x["path"], cfg.get("pester", []))], key=lambda x: x["path"]) + + +def xunit(cfg, tests): + return sorted([x for x in tests["xunit"] if match(x["path"], cfg.get("xunit", []))], key=lambda x: x["path"]) + + +def seeds(cfg, pts): + pre = cfg.get("sec_prefix", []) + return [p for p in pts if any(p["file"].startswith(q) for q in pre)] + + +def tests_rows(cfg, tests): + ps, xs = pester(cfg, tests), xunit(cfg, tests) + rows = [] + if ps: + rows += ["| Pester file | Describe | Context | It |", "|---|---:|---:|---:|"] + for x in ps[:60]: + rows.append("| `%s` | %d | %d | %d |" % (x["path"], x["describe"], x["context"], x["it"])) + if len(ps) > 60: + rows.append("| (%d more files not listed) | | | %d |" % (len(ps) - 60, sum(x["it"] for x in ps[60:]))) + rows.append("| **total (%d files)** | %d | %d | %d |" % (len(ps), sum(x["describe"] for x in ps), sum(x["context"] for x in ps), sum(x["it"] for x in ps))) + else: + rows.append("No Pester file matched the path keywords.") + rows.append("") + if xs: + rows += ["| xUnit file | Facts | Theories | Classes |", "|---|---:|---:|---|"] + for x in xs: + rows.append("| `%s` | %d | %d | %s |" % (x["path"], x["facts"], x["theories"], ", ".join("`%s`" % c for c in x["classes"][:4]))) + else: + rows.append("No xUnit file matched the path keywords.") + rows += ["", "Matching is by path keyword (`APPROX`): %s." % (", ".join("`%s`" % k for k in cfg.get("pester", []) + cfg.get("xunit", [])) or "none")] + return rows + + +def seed_rows(cfg, pts): + s = seeds(cfg, pts) + if not s: + return ["No scanner seed hit under the prefixes of this subsystem (`UNVERIFIED`: absence of a hit is not proof of absence of a check)."] + c = Counter(p["category"] for p in s) + f = Counter(p["file"] for p in s) + rows = ["| Category | Seed hits |", "|---|---:|"] + for k, v in c.most_common(): + rows.append("| %s | %d |" % (k, v)) + rows.append("| **total** | %d |" % len(s)) + rows += ["", "Files with most hits:", "", "| File | Hits |", "|---|---:|"] + for k, v in f.most_common(8): + rows.append("| `%s` | %d |" % (k, v)) + rows += ["", "These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point."] + return rows + + +def main(): + tests, pts = J("tests.json"), J("enforcement-points.json") + if len(sys.argv) >= 3 and sys.argv[1] == "--count": + key, what = sys.argv[2].split(":") + cfg = CFG[key] + v = {"pester_files": len(pester(cfg, tests)), "pester_its": sum(x["it"] for x in pester(cfg, tests)), + "xunit_files": len(xunit(cfg, tests)), "seed_hits": len(seeds(cfg, pts))}[what] + print(v) + return + for key, cfg in CFG.items(): + p = os.path.join(OUT, cfg["doc"]) + if not os.path.exists(p): + continue + fill(p, "tests", tests_rows(cfg, tests)) + fill(p, "secseeds", seed_rows(cfg, pts)) + print("rendered", key, cfg["doc"]) + + +if __name__ == "__main__": + main() From 275fd584de77b5b4eecbdde7a7f5e266c49edc76 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:51:12 +0000 Subject: [PATCH 3/3] Phase 4 documents: hosting, remoting, platform (T41-T49) Add 04-hosting (host abstraction, console host, SDK), 05-remoting (PSRP, transports, CLIXML, session configurations), 06-platform (Windows/Unix split, native interop). Register DEV-HOST/SDK/REM/SER/JEA/PLAT rows, claims, ledger ranges, VB-058..VB-075, D-025..D-027; update PROGRESS and README. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01HnR7RpZuh9mvNMRTtzxTvk --- docs-out/.work/cf/T4.json | 104 +++++ docs-out/.work/claims/T4.jsonl | 12 + docs-out/.work/files-ledger.csv | 58 +-- docs-out/.work/ledger-proposals/T41_T49.csv | 32 ++ docs-out/04-hosting/console-host.md | 193 ++++++++ docs-out/04-hosting/host-abstraction.md | 182 ++++++++ docs-out/04-hosting/sdk.md | 177 +++++++ docs-out/05-remoting/jea.md | 172 +++++++ docs-out/05-remoting/psrp-protocol.md | 200 ++++++++ docs-out/05-remoting/serialization-clixml.md | 175 +++++++ docs-out/05-remoting/transports.md | 188 ++++++++ docs-out/06-platform/native-interop.md | 157 +++++++ docs-out/06-platform/windows-unix-split.md | 203 ++++++++ docs-out/09-synthesis/deviation-register.md | 11 + docs-out/DECISIONS.md | 3 + docs-out/PROGRESS.md | 17 +- docs-out/README.md | 6 + docs-out/VERIFICATION-BACKLOG.md | 18 + docs-out/data/claims/T4.jsonl | 12 + docs-out/data/lessons.json | 462 +++++++++++++++++++ docs-out/data/subject-keys.json | 12 + docs-out/tools/annotations/T3-docs.json | 163 +++++++ 22 files changed, 2524 insertions(+), 33 deletions(-) create mode 100644 docs-out/.work/cf/T4.json create mode 100644 docs-out/.work/claims/T4.jsonl create mode 100644 docs-out/.work/ledger-proposals/T41_T49.csv create mode 100644 docs-out/04-hosting/console-host.md create mode 100644 docs-out/04-hosting/host-abstraction.md create mode 100644 docs-out/04-hosting/sdk.md create mode 100644 docs-out/05-remoting/jea.md create mode 100644 docs-out/05-remoting/psrp-protocol.md create mode 100644 docs-out/05-remoting/serialization-clixml.md create mode 100644 docs-out/05-remoting/transports.md create mode 100644 docs-out/06-platform/native-interop.md create mode 100644 docs-out/06-platform/windows-unix-split.md create mode 100644 docs-out/data/claims/T4.jsonl diff --git a/docs-out/.work/cf/T4.json b/docs-out/.work/cf/T4.json new file mode 100644 index 0000000..174e28b --- /dev/null +++ b/docs-out/.work/cf/T4.json @@ -0,0 +1,104 @@ +{ +"cfs": [], +"deviations": [ +{ +"id": "DEV-HOST-200", +"baseline_behavior": "ConsoleHost parses switches that widen access (for example -noprofile, -executionpolicy, -configurationfile, server modes) from the command line, then starts; policy applies after parsing.", +"proposed_behavior": "Load policy from a signed source first; refuse switches that widen access when lockdown is on; refuse server modes without an authenticated transport.", +"benefit": "Closes a path where a launch argument weakens a locked-down start.", +"compatibility_cost": "Scripts that pass widening switches under lockdown stop working.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-HOST-201", +"baseline_behavior": "ConsoleHost carries Windows-only pieces: ConsoleControl Win32 calls, jump list, -windowstyle, -sta by default, Hyper-V socket modes.", +"proposed_behavior": "Leave them out of the first version.", +"benefit": "Smaller host, fewer native calls.", +"compatibility_cost": "No Windows console extras, no Hyper-V mode.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-SDK-205", +"baseline_behavior": "The default embedded session is full language with all commands visible.", +"proposed_behavior": "Default embedded session is restricted; full language needs an explicit opt-in.", +"benefit": "Safe default for host applications.", +"compatibility_cost": "Embedders must opt in to the full session.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-REM-210", +"baseline_behavior": "PSRP carries XML payloads, fragments, in-protocol key exchange and priority streams over several transports.", +"proposed_behavior": "No remoting in the first version; if added later, use a typed binary protocol with transport-level authentication.", +"benefit": "Removes a large attack surface and a large unverified area.", +"compatibility_cost": "No remote sessions initially.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-REM-211", +"baseline_behavior": "Transports: WSMan (native plugin), SSH subsystem, named pipe, Hyper-V socket, container and VM process.", +"proposed_behavior": "Omit WSMan, Hyper-V and container transports; consider SSH-style process stdio first if remoting is added.", +"benefit": "Avoids Windows-only native components.", +"compatibility_cost": "No WSMan or Hyper-V endpoints.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-SER-215", +"baseline_behavior": "CLIXML reader has depth limits but no explicit maximum input size; depth-1 property bags can include sensitive properties.", +"proposed_behavior": "Add a maximum input size and streaming limit to the reader; mark sensitive properties so shallow bags cannot leak them.", +"benefit": "Bounds memory and leakage.", +"compatibility_cost": "Large documents fail to import unless a limit is raised.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-JEA-220", +"baseline_behavior": "Role merging concatenates values from all matching roles; there is no deny rule.", +"proposed_behavior": "Add explicit deny rules with deny-over-allow precedence and a report of effective capabilities.", +"benefit": "Predictable least privilege.", +"compatibility_cost": "Role files need a deny section; merge results change when deny is used.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-PLAT-230", +"baseline_behavior": "OS differences are spread over 390 conditional blocks and many runtime queries.", +"proposed_behavior": "One platform interface with a runtime capability table; conditional compilation only inside interface implementations.", +"benefit": "Testable on every OS; one place to review.", +"compatibility_cost": "Larger initial design effort.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-PLAT-231", +"baseline_behavior": "Native layer: hand-kept struct layouts for libpsl-native, three interop styles, C source in another repository.", +"proposed_behavior": "One native surface per OS with generated bindings and a layout test in CI.", +"benefit": "Fewer ABI mismatches.", +"compatibility_cost": "Build tooling must generate and verify bindings.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-PLAT-232", +"baseline_behavior": "Last-resort resolver loads /. from the folder of the calling assembly with no signature or location check.", +"proposed_behavior": "Prefer the application directory and a signed or pinned per-RID folder; refuse writable locations unless policy allows.", +"benefit": "Blocks library planting through writable module folders.", +"compatibility_cost": "Modules with native libraries need a signed or allowed location.", +"affected_cfs": [], +"status": "PROPOSED" +}, +{ +"id": "DEV-PLAT-233", +"baseline_behavior": "Zone checks and the execution-policy store exist only on Windows; Unix treats every file as local and cannot set a policy.", +"proposed_behavior": "Define origin and policy for every supported OS, or report 'not enforced' through one status API.", +"benefit": "Security behavior is the same or explicitly different per OS.", +"compatibility_cost": "Unix users get a policy store and origin marks.", +"affected_cfs": [], +"status": "PROPOSED" +} +] +} \ No newline at end of file diff --git a/docs-out/.work/claims/T4.jsonl b/docs-out/.work/claims/T4.jsonl new file mode 100644 index 0000000..19630f1 --- /dev/null +++ b/docs-out/.work/claims/T4.jsonl @@ -0,0 +1,12 @@ +{"id": "CLM-PLT-0001", "subject_key": "platform.cache_dir_unix_default", "predicate": "default cache directory on Unix when XDG_CACHE_HOME is unset", "value": "~/.cache/powershell", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0002", "subject_key": "platform.shared_modules_unix", "predicate": "shared modules path on Unix", "value": "/usr/local/share/powershell/Modules", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0003", "subject_key": "platform.getfolderpath_unix_supported", "predicate": "special folders supported by Platform.GetFolderPath on Unix", "value": "ProgramFiles, MyDocuments", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.GetFolderPath@L396-L408"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0004", "subject_key": "platform.native_lib_name_unix", "predicate": "native library name imported by Platform.Unix.NativeMethods", "value": "libpsl-native", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0005", "subject_key": "platform.native_libraryimport_count", "predicate": "LibraryImport entries in Platform.Unix.NativeMethods", "value": "17", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0006", "subject_key": "platform.native_rid_folders", "predicate": "native resolver folder prefixes", "value": "win-, linux-, osx- plus lower-case process architecture", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext@L19-L565"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0007", "subject_key": "platform.pinvoke_dll_name_constants", "predicate": "string constants in PinvokeDllNames", "value": "125", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/utils/PInvokeDllNames.cs::PinvokeDllNames@L13-L140"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0008", "subject_key": "platform.interop_windows_files", "predicate": "source files under engine/Interop/Windows", "value": "33", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Interop/Windows/CreateHardLink.cs"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0009", "subject_key": "platform.native_package", "predicate": "NuGet package that supplies native components", "value": "Microsoft.PowerShell.Native 700.0.2", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/System.Management.Automation.csproj@L44-L46"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0010", "subject_key": "platform.unix_nonzero_exit_hex_width", "predicate": "hex width of the exit code text on Unix", "value": "2 digits (0x{0:X2}); Windows uses 8 digits", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/NativeCommandProcessor.cs@L1176-L1200"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0011", "subject_key": "platform.unix_origin_check", "predicate": "IsLocalFile result on Unix", "value": "always true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/security/SecurityManager.cs@L416-L433"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0012", "subject_key": "platform.unix_set_execution_policy", "predicate": "SetExecutionPolicy on Unix", "value": "throws PlatformNotSupportedException", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/security/SecuritySupport.cs@L136-L145"], "doc": "PLAT-SPLIT-001", "cf": null} diff --git a/docs-out/.work/files-ledger.csv b/docs-out/.work/files-ledger.csv index b252fd7..41666ed 100644 --- a/docs-out/.work/files-ledger.csv +++ b/docs-out/.work/files-ledger.csv @@ -1218,7 +1218,7 @@ src/Microsoft.PowerShell.ConsoleHost/WindowsTaskbarJumpList/PropertyKey.cs,120,B src/Microsoft.PowerShell.ConsoleHost/WindowsTaskbarJumpList/TaskbarJumpList.cs,144,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs,1715,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleControl.cs,3189,B,,,unread -src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs,3159,B,,,unread +src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs,3159,B,100-344;484-560;1338-1350;1576-1710;1748-2200;2600-2830,HOST-CONSOLE-001;PLAT-SPLIT-001,partial src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostRawUserInterface.cs,1693,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostTranscript.cs,130,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs,2268,B,,,unread @@ -1226,10 +1226,10 @@ src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterfaceProgress.c src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterfacePrompt.cs,560,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterfacePromptForChoice.cs,467,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterfaceSecurity.cs,126,B,,,unread -src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs,81,B,,,unread +src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs,81,B,1-81,HOST-CONSOLE-001,read src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleTextWriter.cs,92,B,,,unread -src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs,709,B,,,unread -src/Microsoft.PowerShell.ConsoleHost/host/msh/ManagedEntrance.cs,124,B,,,unread +src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs,709,B,150-260;330-400,HOST-CONSOLE-001,partial +src/Microsoft.PowerShell.ConsoleHost/host/msh/ManagedEntrance.cs,124,B,1-124,HOST-CONSOLE-001,read src/Microsoft.PowerShell.ConsoleHost/host/msh/PendingProgress.cs,994,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ProgressNode.cs,611,B,,,unread src/Microsoft.PowerShell.ConsoleHost/host/msh/ProgressPane.cs,349,B,,,unread @@ -1531,8 +1531,8 @@ src/PowerShell.Core.Instrumentation/RegisterManifest.ps1,97,C,,,unread src/ResGen/Program.cs,189,C,,,unread src/System.Management.Automation/AssemblyInfo.cs,16,B,,,unread src/System.Management.Automation/GlobalSuppressions.cs,8,B,,,unread -src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs,626,B,,,unread -src/System.Management.Automation/CoreCLR/CorePsPlatform.cs,1125,B,,,unread +src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs,626,B,98-108;205-260;300-370;539-566,PLAT-NATIVE-001;PLAT-SPLIT-001,partial +src/System.Management.Automation/CoreCLR/CorePsPlatform.cs,1125,B,1-1125,PLAT-NATIVE-001;PLAT-SPLIT-001,read src/System.Management.Automation/CoreCLR/CorePsStub.cs,874,B,,,unread src/System.Management.Automation/CoreCLR/EventResource.cs,626,B,,,unread src/System.Management.Automation/DscSupport/CimDSCParser.cs,4064,B,,,unread @@ -1632,7 +1632,7 @@ src/System.Management.Automation/engine/CommandParameter.cs,230,B,,,unread src/System.Management.Automation/engine/CommandPathSearch.cs,596,B,,,unread src/System.Management.Automation/engine/CommandProcessor.cs,869,A,1-869,RT-BIND-001;RT-PIPELINE-001,read src/System.Management.Automation/engine/CommandProcessorBase.cs,1044,A,20-1044,RT-CMDLET-001;RT-PIPELINE-001,partial -src/System.Management.Automation/engine/CommandSearcher.cs,1733,A,22-1100;1322-1640,RT-DISCOVERY-001,partial +src/System.Management.Automation/engine/CommandSearcher.cs,1733,A,22-1100;1322-1640,PLAT-SPLIT-001;RT-DISCOVERY-001,partial src/System.Management.Automation/engine/CommonCommandParameters.cs,284,B,,,unread src/System.Management.Automation/engine/CompiledCommandParameter.cs,683,A,,,unread src/System.Management.Automation/engine/ConfigurationInfo.cs,172,B,,,unread @@ -1661,7 +1661,7 @@ src/System.Management.Automation/engine/FunctionInfo.cs,497,B,,,unread src/System.Management.Automation/engine/GetCommandCommand.cs,1743,B,,,unread src/System.Management.Automation/engine/ICommandRuntime.cs,622,B,,,unread src/System.Management.Automation/engine/InformationRecord.cs,264,B,,,unread -src/System.Management.Automation/engine/InitialSessionState.cs,5674,B,4655-4685;5515-5535,RT-DISCOVERY-001;RT-PROVIDER-001,partial +src/System.Management.Automation/engine/InitialSessionState.cs,5674,B,1310-1400;1498-1606;1749-1850;4655-4685;5515-5535,HOST-ABSTRACTION-001;REM-JEA-001;RT-DISCOVERY-001;RT-PROVIDER-001,partial src/System.Management.Automation/engine/InternalCommands.cs,2752,B,240-271;377-470,RT-JOBS-001,partial src/System.Management.Automation/engine/InvocationInfo.cs,537,B,,,unread src/System.Management.Automation/engine/ItemCmdletProviderInterfaces.cs,2107,B,,,unread @@ -1680,13 +1680,13 @@ src/System.Management.Automation/engine/MshSnapinQualifiedName.cs,134,B,,,unread src/System.Management.Automation/engine/NativeCommand.cs,36,B,,,unread src/System.Management.Automation/engine/NativeCommandParameterBinder.cs,520,A,58-470,RT-DISCOVERY-001,partial src/System.Management.Automation/engine/NativeCommandParameterBinderController.cs,106,A,,,unread -src/System.Management.Automation/engine/NativeCommandProcessor.cs,2651,A,197-232;557-606;715-967;1082-1240;1449-1514;1586-1884,RT-DISCOVERY-001,partial +src/System.Management.Automation/engine/NativeCommandProcessor.cs,2651,A,197-232;557-606;715-967;1082-1240;1408-1430;1449-1520;1586-1921,PLAT-SPLIT-001;RT-DISCOVERY-001,partial src/System.Management.Automation/engine/NullString.cs,45,B,,,unread src/System.Management.Automation/engine/ObjectEventRegistrationBase.cs,232,B,,,unread src/System.Management.Automation/engine/OrderedHashtable.cs,236,B,,,unread src/System.Management.Automation/engine/PSClassInfo.cs,86,B,,,unread src/System.Management.Automation/engine/PSClassSearcher.cs,329,B,,,unread -src/System.Management.Automation/engine/PSConfiguration.cs,794,B,,,unread +src/System.Management.Automation/engine/PSConfiguration.cs,794,B,250-262;375-383,PLAT-SPLIT-001,partial src/System.Management.Automation/engine/PSVersionInfo.cs,1038,B,,,unread src/System.Management.Automation/engine/ParameterBinderBase.cs,2067,A,329-667,RT-BIND-001,partial src/System.Management.Automation/engine/ParameterBinderController.cs,1326,A,133-330;514-870;919-1190,RT-BIND-001,partial @@ -1754,7 +1754,7 @@ src/System.Management.Automation/engine/pipeline.cs,1668,A,1-1668,RT-ERRORS-001; src/System.Management.Automation/engine/regex.cs,1384,B,,,unread src/System.Management.Automation/engine/scriptparameterbinder.cs,170,A,,,unread src/System.Management.Automation/engine/scriptparameterbindercontroller.cs,231,A,,,unread -src/System.Management.Automation/engine/serialization.cs,7679,B,,,unread +src/System.Management.Automation/engine/serialization.cs,7679,B,34-220;335-345;971-1100;2295-2388;3656-3757;4172-4215;4792-4810;4910-4975;5167-5240;6670-6740,REM-CLIXML-001,partial src/System.Management.Automation/engine/COM/ComAdapter.cs,371,B,,,unread src/System.Management.Automation/engine/COM/ComDispatch.cs,42,B,,,unread src/System.Management.Automation/engine/COM/ComInvoker.cs,437,B,,,unread @@ -1831,10 +1831,10 @@ src/System.Management.Automation/engine/ExperimentalFeature/GetExperimentalFeatu src/System.Management.Automation/engine/Interop/Windows/AllocConsole.cs,16,B,,,unread src/System.Management.Automation/engine/Interop/Windows/AssignProcessToJobObject.cs,19,B,,,unread src/System.Management.Automation/engine/Interop/Windows/CloseHandle.cs,16,B,,,unread -src/System.Management.Automation/engine/Interop/Windows/CoInitializeEx.cs,18,B,,,unread +src/System.Management.Automation/engine/Interop/Windows/CoInitializeEx.cs,18,B,1-18,PLAT-NATIVE-001,read src/System.Management.Automation/engine/Interop/Windows/CoUninitialize.cs,15,B,,,unread src/System.Management.Automation/engine/Interop/Windows/CreateFile.cs,95,B,,,unread -src/System.Management.Automation/engine/Interop/Windows/CreateHardLink.cs,16,B,,,unread +src/System.Management.Automation/engine/Interop/Windows/CreateHardLink.cs,16,B,1-16,PLAT-NATIVE-001,read src/System.Management.Automation/engine/Interop/Windows/CreateIoCompletionPort.cs,38,B,,,unread src/System.Management.Automation/engine/Interop/Windows/CreateJobObject.cs,30,B,,,unread src/System.Management.Automation/engine/Interop/Windows/CreateSymbolicLink.cs,25,B,,,unread @@ -1900,13 +1900,13 @@ src/System.Management.Automation/engine/hostifaces/FieldDescription.cs,416,B,,,u src/System.Management.Automation/engine/hostifaces/History.cs,1902,B,,,unread src/System.Management.Automation/engine/hostifaces/HostUtilities.cs,605,B,,,unread src/System.Management.Automation/engine/hostifaces/InformationalRecord.cs,267,B,,,unread -src/System.Management.Automation/engine/hostifaces/InternalHost.cs,581,B,,,unread +src/System.Management.Automation/engine/hostifaces/InternalHost.cs,581,B,225-330,HOST-ABSTRACTION-001,partial src/System.Management.Automation/engine/hostifaces/InternalHostRawUserInterface.cs,615,B,,,unread src/System.Management.Automation/engine/hostifaces/InternalHostUserInterface.cs,1033,B,,,unread src/System.Management.Automation/engine/hostifaces/ListModifier.cs,300,B,,,unread src/System.Management.Automation/engine/hostifaces/LocalConnection.cs,1569,B,128-175;591-730,RT-PIPELINE-001,partial src/System.Management.Automation/engine/hostifaces/LocalPipeline.cs,1400,B,21-1147;1266-1400,RT-PIPELINE-001,partial -src/System.Management.Automation/engine/hostifaces/MshHost.cs,328,B,,,unread +src/System.Management.Automation/engine/hostifaces/MshHost.cs,328,B,222-327,HOST-ABSTRACTION-001,partial src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs,1915,B,,,unread src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs,1480,B,418-470;1190-1260,RT-ERRORS-001,partial src/System.Management.Automation/engine/hostifaces/PSCommand.cs,517,B,,,unread @@ -2035,11 +2035,11 @@ src/System.Management.Automation/engine/remoting/common/ObjectRef.cs,91,B,,,unre src/System.Management.Automation/engine/remoting/common/PSETWTracer.cs,281,B,,,unread src/System.Management.Automation/engine/remoting/common/PSSessionConfigurationTypeOption.cs,103,B,,,unread src/System.Management.Automation/engine/remoting/common/RemoteSessionHyperVSocket.cs,1157,B,,,unread -src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs,1318,B,,,unread -src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs,3803,B,,,unread +src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs,1318,B,426-500;651-700,REM-TRANSPORT-001,partial +src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs,3803,B,1928-2070;2193-2341;2355-2360;2401-2401;2490-2525;2585-2600,PLAT-NATIVE-001;REM-TRANSPORT-001,partial src/System.Management.Automation/engine/remoting/common/RunspaceInitInfo.cs,34,B,,,unread src/System.Management.Automation/engine/remoting/common/RunspacePoolStateInfo.cs,41,B,,,unread -src/System.Management.Automation/engine/remoting/common/fragmentor.cs,1126,B,,,unread +src/System.Management.Automation/engine/remoting/common/fragmentor.cs,1126,B,29-280;965-1125,REM-PSRP-001,partial src/System.Management.Automation/engine/remoting/common/misc.cs,418,B,,,unread src/System.Management.Automation/engine/remoting/common/psstreamobject.cs,508,B,,,unread src/System.Management.Automation/engine/remoting/common/remotesession.cs,34,B,,,unread @@ -2050,10 +2050,10 @@ src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteDeb src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHost.cs,762,B,,,unread src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHostEncoder.cs,783,B,,,unread src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteSessionCapability.cs,406,B,,,unread -src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemotingDataObject.cs,309,B,,,unread -src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs,1608,B,,,unread -src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs,3031,B,,,unread -src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs,2680,B,,,unread +src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemotingDataObject.cs,309,B,150-309,REM-PSRP-001,partial +src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs,1608,B,194-225,REM-PSRP-001;REM-TRANSPORT-001,partial +src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs,3031,B,870-1015;1781-2200;2475-2680,REM-JEA-001,partial +src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs,2680,B,37-260,REM-TRANSPORT-001,partial src/System.Management.Automation/engine/remoting/fanin/PSPrincipal.cs,386,B,,,unread src/System.Management.Automation/engine/remoting/fanin/PSSessionConfigurationData.cs,244,B,,,unread src/System.Management.Automation/engine/remoting/fanin/PriorityCollection.cs,933,B,,,unread @@ -2075,7 +2075,7 @@ src/System.Management.Automation/engine/remoting/server/ServerRunspacePoolDriver src/System.Management.Automation/engine/remoting/server/ServerSteppablePipelineDriver.cs,477,B,,,unread src/System.Management.Automation/engine/remoting/server/ServerSteppablePipelineSubscriber.cs,297,B,,,unread src/System.Management.Automation/engine/remoting/server/WSManChannelEvents.cs,70,B,,,unread -src/System.Management.Automation/engine/remoting/server/serverremotesession.cs,1095,B,,,unread +src/System.Management.Automation/engine/remoting/server/serverremotesession.cs,1095,B,300-360;967-1058,REM-PSRP-001,partial src/System.Management.Automation/engine/remoting/server/serverremotesessionstatemachine.cs,1023,B,,,unread src/System.Management.Automation/engine/remoting/server/serverremotingprotocol.cs,100,B,,,unread src/System.Management.Automation/engine/remoting/server/serverremotingprotocolimplementation.cs,244,B,,,unread @@ -2141,7 +2141,7 @@ src/System.Management.Automation/namespaces/CoreCommandContext.cs,1148,B,,,unrea src/System.Management.Automation/namespaces/DriveProviderBase.cs,238,B,,,unread src/System.Management.Automation/namespaces/EnvironmentProvider.cs,247,B,,,unread src/System.Management.Automation/namespaces/FileSystemContentStream.cs,1503,B,,,unread -src/System.Management.Automation/namespaces/FileSystemProvider.cs,9605,B,,,unread +src/System.Management.Automation/namespaces/FileSystemProvider.cs,9605,B,595-606;1348-1385;2393-2423;5000-5010,PLAT-SPLIT-001,partial src/System.Management.Automation/namespaces/FileSystemSecurity.cs,344,B,,,unread src/System.Management.Automation/namespaces/FunctionProvider.cs,389,B,,,unread src/System.Management.Automation/namespaces/IContentProvider.cs,149,B,,,unread @@ -3219,15 +3219,15 @@ src/System.Management.Automation/security/CatalogHelper.cs,841,B,,,unread src/System.Management.Automation/security/CredentialParameter.cs,91,B,,,unread src/System.Management.Automation/security/MshSignature.cs,393,B,,,unread src/System.Management.Automation/security/SecureStringHelper.cs,643,B,,,unread -src/System.Management.Automation/security/SecurityManager.cs,828,B,,,unread -src/System.Management.Automation/security/SecuritySupport.cs,1777,B,,,unread +src/System.Management.Automation/security/SecurityManager.cs,828,B,410-435,PLAT-SPLIT-001,partial +src/System.Management.Automation/security/SecuritySupport.cs,1777,B,132-145,PLAT-SPLIT-001,partial src/System.Management.Automation/security/nativeMethods.cs,1513,B,,,unread src/System.Management.Automation/security/wldpNativeMethods.cs,950,B,,,unread src/System.Management.Automation/security/Win32Native/WinTrust.cs,440,B,,,unread src/System.Management.Automation/singleshell/config/MshConsoleLoadException.cs,116,B,,,unread src/System.Management.Automation/singleshell/config/MshSnapinInfo.cs,1330,B,,,unread src/System.Management.Automation/singleshell/config/MshSnapinLoadException.cs,183,B,,,unread -src/System.Management.Automation/utils/ClrFacade.cs,371,B,,,unread +src/System.Management.Automation/utils/ClrFacade.cs,371,B,116-130,PLAT-SPLIT-001,partial src/System.Management.Automation/utils/CommandDiscoveryExceptions.cs,477,B,,,unread src/System.Management.Automation/utils/CommandProcessorExceptions.cs,100,B,,,unread src/System.Management.Automation/utils/CoreProviderCmdlets.cs,197,B,,,unread @@ -3253,7 +3253,7 @@ src/System.Management.Automation/utils/MshTraceSource.cs,538,B,,,unread src/System.Management.Automation/utils/ObjectReader.cs,824,B,,,unread src/System.Management.Automation/utils/ObjectStream.cs,1943,B,,,unread src/System.Management.Automation/utils/ObjectWriter.cs,231,B,,,unread -src/System.Management.Automation/utils/PInvokeDllNames.cs,141,B,,,unread +src/System.Management.Automation/utils/PInvokeDllNames.cs,141,B,1-141,PLAT-NATIVE-001,read src/System.Management.Automation/utils/PSTelemetryMethods.cs,534,B,,,unread src/System.Management.Automation/utils/PSTelemetryWrapper.cs,150,B,,,unread src/System.Management.Automation/utils/ParameterBinderExceptions.cs,1031,B,,,unread @@ -3262,7 +3262,7 @@ src/System.Management.Automation/utils/PathUtils.cs,907,B,,,unread src/System.Management.Automation/utils/PlatformInvokes.cs,679,B,,,unread src/System.Management.Automation/utils/PowerShellETWTracer.cs,1540,B,,,unread src/System.Management.Automation/utils/PowerShellExecutionHelper.cs,198,B,,,unread -src/System.Management.Automation/utils/PsUtils.cs,562,B,,,unread +src/System.Management.Automation/utils/PsUtils.cs,562,B,103-110,PLAT-SPLIT-001,partial src/System.Management.Automation/utils/ResourceManagerCache.cs,244,B,,,unread src/System.Management.Automation/utils/RuntimeException.cs,299,B,,,unread src/System.Management.Automation/utils/SessionStateExceptions.cs,971,B,,,unread diff --git a/docs-out/.work/ledger-proposals/T41_T49.csv b/docs-out/.work/ledger-proposals/T41_T49.csv new file mode 100644 index 0000000..9aaafa6 --- /dev/null +++ b/docs-out/.work/ledger-proposals/T41_T49.csv @@ -0,0 +1,32 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/hostifaces/MshHost.cs,222-327,HOST-ABSTRACTION-001 +src/System.Management.Automation/engine/hostifaces/InternalHost.cs,225-330,HOST-ABSTRACTION-001 +src/System.Management.Automation/engine/InitialSessionState.cs,1310-1400;1498-1606;1749-1850,HOST-ABSTRACTION-001;REM-JEA-001 +src/Microsoft.PowerShell.ConsoleHost/host/msh/ManagedEntrance.cs,1-124,HOST-CONSOLE-001 +src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs,1-81,HOST-CONSOLE-001 +src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs,100-344;484-560;1338-1350;1576-1710;1748-2200;2600-2830,HOST-CONSOLE-001 +src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs,1338-1350;2028-2042;2650-2660,PLAT-SPLIT-001 +src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs,150-260;330-400,HOST-CONSOLE-001 +src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemotingDataObject.cs,150-309,REM-PSRP-001 +src/System.Management.Automation/engine/remoting/common/fragmentor.cs,29-280;965-1125,REM-PSRP-001 +src/System.Management.Automation/engine/remoting/server/serverremotesession.cs,300-360;967-1058,REM-PSRP-001 +src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs,194-225,REM-PSRP-001;REM-TRANSPORT-001 +src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs,37-260,REM-TRANSPORT-001 +src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs,1928-2070;2193-2341;2355-2360;2401-2401;2490-2525;2585-2600,REM-TRANSPORT-001 +src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs,2355-2360;2401-2401;2490-2525;2585-2600,PLAT-NATIVE-001 +src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs,426-500;651-700,REM-TRANSPORT-001 +src/System.Management.Automation/engine/serialization.cs,34-220;335-345;971-1100;2295-2388;3656-3757;4172-4215;4792-4810;4910-4975;5167-5240;6670-6740,REM-CLIXML-001 +src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs,870-1015;1781-2200;2475-2680,REM-JEA-001 +src/System.Management.Automation/CoreCLR/CorePsPlatform.cs,1-1125,PLAT-SPLIT-001;PLAT-NATIVE-001 +src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs,98-108;205-260;300-370;539-566,PLAT-SPLIT-001;PLAT-NATIVE-001 +src/System.Management.Automation/engine/NativeCommandProcessor.cs,798-830;1176-1200;1408-1430;1492-1520;1855-1921,PLAT-SPLIT-001 +src/System.Management.Automation/namespaces/FileSystemProvider.cs,595-606;1348-1385;2393-2423;5000-5010,PLAT-SPLIT-001 +src/System.Management.Automation/security/SecurityManager.cs,410-435,PLAT-SPLIT-001 +src/System.Management.Automation/security/SecuritySupport.cs,132-145,PLAT-SPLIT-001 +src/System.Management.Automation/engine/PSConfiguration.cs,250-262;375-383,PLAT-SPLIT-001 +src/System.Management.Automation/utils/PsUtils.cs,103-110,PLAT-SPLIT-001 +src/System.Management.Automation/utils/ClrFacade.cs,116-130,PLAT-SPLIT-001 +src/System.Management.Automation/engine/CommandSearcher.cs,1340-1358,PLAT-SPLIT-001 +src/System.Management.Automation/utils/PInvokeDllNames.cs,1-141,PLAT-NATIVE-001 +src/System.Management.Automation/engine/Interop/Windows/CoInitializeEx.cs,1-18,PLAT-NATIVE-001 +src/System.Management.Automation/engine/Interop/Windows/CreateHardLink.cs,1-16,PLAT-NATIVE-001 diff --git a/docs-out/04-hosting/console-host.md b/docs-out/04-hosting/console-host.md new file mode 100644 index 0000000..131e0a4 --- /dev/null +++ b/docs-out/04-hosting/console-host.md @@ -0,0 +1,193 @@ +--- +id: HOST-CONSOLE-001 +title: Console host +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/Microsoft.PowerShell.ConsoleHost/host/msh/ManagedEntrance.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs] +mirror_cards: [src-mirror/src/Microsoft.PowerShell.ConsoleHost/_project.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json, data/projects.json] +--- + +# Console host + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. Contract implemented: [host-abstraction](host-abstraction.md). Embedding the engine without this host: [sdk](sdk.md). The `pwsh` executable that calls into this assembly is a separate native launcher (Phase 6 platform notes). + +## 1. Purpose and scope [SPEC] + +`Microsoft.PowerShell.ConsoleHost` implements the `pwsh` command-line shell: argument parsing, runspace creation, profile loading, running `-Command`/`-File` content, the interactive read-evaluate-print loop, break handling, the server modes used by remoting (`-s`, SSH, named pipe, sockets), and exit codes. It also holds the console UI classes. External dependencies not in this repository: the **PSReadLine** module (line editing, loaded as a module; version 2.4.5 pinned in `src/Modules/PSGalleryModules.csproj`), the native launcher, and the OS console. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ManagedEntrance.cs::UnmanagedPSEntry@L21-L123 | class | entry from the native launcher (`Start(args, argc)`) | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs::ConsoleShell@L15-L80 | static class | public embedding entry (`Start(bannerText, helpText, args)` and an overload taking an `InitialSessionState`) | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost@L42-L3121 | class | the `PSHost`; startup, runspace loop, break handling, nested prompt | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::InputLoop@L2493-L3031 | nested class | the prompt loop | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs::CommandLineParameterParser@L156-L1714 | class | command-line parsing | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor@L25-L708 | class | runs a command or pipeline for the host, adding `Out-Default` | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs::ConsoleHostUserInterface@L30-L2267 | class | `PSHostUserInterface` over the console | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostRawUserInterface.cs::ConsoleHostRawUserInterface@L24-L1301 | class | `PSHostRawUserInterface` over the console | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleControl.cs::ConsoleControl@L47-L3188 | class | Win32 console calls (Windows) | B | +| src/Microsoft.PowerShell.ConsoleHost/host/msh/*Progress*.cs, ProgressPane.cs, PendingProgress.cs | classes | progress rendering | B | + +## 3. Responsibilities and non-responsibilities + +Owns: process-level startup and shutdown policy, the interactive loop, console UI. Does not own: the engine, line editing (PSReadLine), the remoting protocols behind the server modes (T5xx), the native launcher. + +## 4. Key types and contracts [IMPL] + +- **Entry**: `UnmanagedPSEntry.Start(args, argc)` runs early startup, sets thread culture from the native resolver, parses the command line (`ConsoleHost.ParseCommandLine`), logs startup, builds the banner, sets `DefaultInitialSessionState = InitialSessionState.CreateDefault2()` and calls `ConsoleHost.Start(banner, help, issProvidedExternally: false)`; a `HostException` caused by Win32 error 6 or 1236 returns the current exit code, any other exception fails fast. `ConsoleShell.Start` lets an embedder supply its own `InitialSessionState` (then a `-ConfigurationFile` on the command line conflicts and aborts). +- **Command-line switches** (case-insensitive, unambiguous prefixes accepted through `MatchSwitch(key, name, shortest)`): `-version`/`-v`, `-help`/`-h`/`-?`, `-login`/`-l`, `-noexit`, `-noprofile`, `-nologo`, `-noninteractive`, `-interactive`/`-i`, `-noprofileloadtime`, `-command`/`-c`, `-commandwithargs`/`-cwa`, `-encodedcommand`/`-e`/`-ec`, `-encodedarguments`/`-ea`, `-file`/`-f`, `-executionpolicy`/`-ep`, `-inputformat`/`-if`, `-outputformat`/`-of`, `-sta`, `-mta`, `-windowstyle`/`-w`, `-workingdirectory`/`-wd`, `-configurationfile`, `-configurationname`/`-config`, `-custompipename`, `-settingsfile`, `-removeworkingdirectorytrailingcharacter`, and the server-mode switches `-servermode`/`-s`, `-sshservermode`, `-namedpipeservermode`, `-socketservermode`, `-v2socketservermode` (Windows) with `-token` and `-utctimestamp`; `-isswait` is debug-only (src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs::CommandLineParameterParser@L156-L1714, switch chain lines 914–1280). Execution policy values accepted: default, remotesigned, bypass, allsigned, restricted, unrestricted, undefined. +- **`ConsoleHost.Start`** (src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.Start@L100-L344): (1) reject a configuration file when the session state was supplied externally; (2) prepend the PowerShell executable's folder to `PATH`; (3) set the JIT profile-optimization root to the cache directory; (4) create the singleton host (a `HostException` is deferred); (5) show help/banner on parse errors; `-version` prints `PowerShell ` and exits 0; (6) at most one server mode may be given, else exit code **64**; a server mode combined with the file-only-entry policy exits 64; (7) dispatch a server mode and return 0: `-s` and SSH → `StdIOProcessMediator.Run` (SSH combines stderr with stdout), named pipe → `RemoteSessionNamedPipeServer.RunServerMode`, socket modes → `HyperVSocketMediator.Run` (the V2 variant requires a token and timestamp, else 64); (8) otherwise load the PSReadLine decision, bind the break handler and call `Run`; (9) in `finally` dispose the host (on Unix after switching off application-cursor-key mode). +- **`Run` and `DoRunspaceLoop`**: copies output/input formats, no-prompt, no-exit and stdin options; on Windows applies `-ExecutionPolicy` as the *process* scope policy; creates a custom named-pipe server when requested; runs the runspace loop: create runspace (`CreateRunspace` → `DoCreateRunspace`), if `-NoExit` enter a nested prompt else mark the session ended; compute the exit code: if `exit N` was requested use N, else evaluate `$global:?` — success gives 0, failure gives 1 (`0 | 1`); close the runspace; a single pass for STA mode. +- **Runspace creation** (`DoCreateRunspace`): with `-ConfigurationFile` the default session state is replaced by one created from that session configuration file (validated); when no custom configuration is given and `LoadPSReadline()` is true — `(no initial command and no file, or -NoExit) and not -NonInteractive` — the `PSReadLine` module is imported into the initial session state; if opening fails with PSReadLine the host retries without it and prints a message; the runspace uses `ThreadOptions.ReuseThread` (and STA on Windows desktop when requested); it becomes the primary runspace. +- **Initialization** (`DoRunspaceInitialization`): enable local and remote script debugging with the host's `DebuggerStop` handler; set the working directory (`Set-Location -LiteralPath`) if requested; if `-ConfigurationName` is given, create a configured *remote* runspace and push it; otherwise enforce system lockdown language mode, print a banner line for constrained/no/restricted language, define `$PROFILE` and — unless `-NoProfile` — dot-source the four profiles **in order**: all-users all-hosts, all-users host-specific, current-user all-hosts, current-user host-specific (a missing file is skipped; an error is reported and does not stop the others; a load time over 500 ms prints a notice unless suppressed); then run `-File` (a `.ps1`, or on Unix any file read as script text), else `-Command`/`-EncodedCommand` text; parameters after `-File` are passed as command parameters; a failure sets exit code 1 when the host exits after the command. +- **Exit codes** (constants): success 0; initialization failure 70 (`ExitCodeInitFailure`, "internal software error"); bad command line 64 (`ExitCodeBadCommandLineParameter`, "usage error"); Ctrl+Break 149 (`128 + 21`); `exit N` returns N; script failure returns 1. +- **Interactive loop** (`InputLoop.Run`): while the session has not ended: unless no-prompt, write the prompt — `>> ` in continuation (block) mode, else (ensuring the cursor is at column 0, evaluating feedback providers after a non-empty response) the output of the `prompt` function (or the debug prompt in debug mode); read a line with tab completion (`ReadLineWithTabCompletion`; with PSReadLine loaded the module takes over line editing); a null line ends the session when input is redirected; an empty line ends block mode and executes the block, else is ignored (except in debug mode); in debug mode the line is a debugger command; otherwise the line is executed through `Executor` (asynchronously when the host is running async and not nested); errors are reported, and an *incomplete parse* error switches to block mode to read more lines (src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::InputLoop.Run@L2600-L2824). +- **`Executor`**: builds pipelines for the host; with the `AddOutputter` option and text output format, a single-command pipeline gets its error stream merged into output and `Out-Default` appended; for multi-statement input `Out-Default` is inserted at the end of each statement, and at the end if the last command is not already `Out-Default`; input objects can be read from redirected stdin through a deserializer when the script uses `$input` (`ReadInputObjects`); the host waits on the pipeline-finished handle and reports a failed pipeline's reason (src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs::Executor@L25-L708). +- **Break handling**: Ctrl+C starts a handler thread that stops the running pipeline (`shouldEndSession: false`); Ctrl+Break breaks into the debugger when interactive, or ends the session when non-interactive; window close/shutdown ends the session; logoff is accepted (`UNVERIFIED` result) ([src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.MyBreakHandler@L484-L524], Windows). + +## 5. Data model + +Host state: singleton instance, the command-line parameters (`s_cpp`), the default initial session state, a global lock, the runspace reference (`RunspaceRef`, which can point at a pushed remote runspace), exit code, flags (`ShouldEndSession`, no-exit, running async, nested), a stack of input loops for nested prompts and a break-handler thread. UI state: console colors, progress panes, input buffer. + +## 6. Control flow + +```mermaid +flowchart TD + L[native launcher] --> E[UnmanagedPSEntry.Start] + E --> P[ParseCommandLine] + P --> S{server mode?} + S -- yes --> M[StdIO, NamedPipe or Socket mediator; exit 0] + S -- no --> R[Run] + R --> RL[DoRunspaceLoop] + RL --> CR[CreateRunspace: ISS, PSReadLine] + CR --> RI[DoRunspaceInitialization: cwd, profiles, -File or -Command] + RI --> N{NoExit?} + N -- yes --> IL[nested prompt: InputLoop] + N -- no --> X[compute exit code from exit request or $?] + IL --> X + X --> CL[close runspace, dispose host] +``` + +## 7. Algorithms + +Exit-code rule: `exit N` wins; otherwise the last statement's success (`$?` evaluated in the global scope) maps to 0 or 1; init failure 70; usage errors 64. Prompt rule: continuation prompt `>> `; ensure column 0; evaluate feedback after a non-empty response. Profile order as above. + +## 8. Observable behavior [SPEC] + +- `pwsh -c "..."` runs the text and exits with 0/1 (or the `exit` value); `pwsh -f script.ps1 args` runs the script with args; `pwsh` with no arguments is interactive with PSReadLine; `-NoExit` stays in a nested prompt after the command; `-NoProfile` skips all four profiles; `-NonInteractive` fails prompts and never loads PSReadLine. +- Ctrl+C stops the running command; Ctrl+Break enters the debugger (interactive). +- Output of commands is formatted by `Out-Default`; error output is merged into output for the host's pipeline. +- Tests: §16 (`ConsoleHost.Tests.ps1` has 89 `It` blocks, `test_CommandLineParser.cs` 55 facts and 52 theories). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Startup errors raise `ConsoleHostStartupException` ("shell cannot be started") with the inner cause; profile errors are reported and ignored; a failing `-File`/`-Command` is reported through `ReportException` and sets the exit code; unhandled exceptions are handled by a global handler that reports through the host (`UnhandledExceptionHandler`); `Win32` errors 6 (invalid handle) and 1236 (connection aborted) at startup are treated as normal terminations; when PSReadLine fails to load, the host continues without it; mutually exclusive server modes exit 64. + +## 10. Concurrency and threading + +The main thread is named "ConsoleHost main thread"; the engine runs commands on its pipeline thread (`ReuseThread`); Ctrl+C/Ctrl+Break handlers run on the console control thread and spawn a separate handler thread; `hostGlobalLock` guards shared host state; the host may run commands asynchronously when `IsRunningAsync` so that break handling can interrupt a command; progress output is serialized by the UI. + +## 11. Extension points and customization hooks + +Profiles (`$PROFILE`, four locations), `-ConfigurationFile`/`-ConfigurationName` (session configurations), `prompt` function, PSReadLine settings, `PSStyle`, `$env:` settings read at startup, the settings file (`-SettingsFile`, `powershell.config.json` keys, see `PSConfiguration`), policy for file-only entry, embedding via `ConsoleShell.Start(initialSessionState, …)`. + +## 12. Dependencies + +Inbound: the native launcher, global-tool shim, `dotnet` hosting. Outbound: `System.Management.Automation` (host, runspace, session state), PSReadLine (module), OS console APIs (`ConsoleControl` on Windows, terminal escape codes on Unix), remoting mediators in the engine assembly (server modes), telemetry (`ApplicationInsightsTelemetry`, startup events). + +## 13. Performance notes + +Start-up uses profile optimization (JIT profile roots by mode: interactive, non-interactive, server modes), `CreateDefault2` (lighter than `CreateDefault`), a startup-time measurement for profiles, and the early-startup initialization class. Non-interactive runs skip PSReadLine. Profile load time over 500 ms triggers a notice. + +## 14. Security considerations + +The host is where process-level policy is applied: `-ExecutionPolicy` (Windows, process scope), system lockdown language mode enforced after runspace creation (`Utils.EnforceSystemLockDownLanguageMode`), a banner for constrained/no/restricted language, the *file-only entry* policy that forbids server modes, configuration files that create restricted sessions, profile scripts that run with the user's rights before any command (an attack and persistence surface), and `-EncodedCommand` (base64 command, a common abuse vector, logged by engine script-block logging). The Unix `-File` path executes non-`.ps1` files as script text. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| logging | 25 | +| execution-policy | 19 | +| securestring | 14 | +| language-mode | 7 | +| jea | 4 | +| app-control | 2 | +| **total** | 71 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs` | 21 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs` | 12 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/StartTranscriptCmdlet.cs` | 11 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs` | 9 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostTranscript.cs` | 7 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/StopTranscriptCmdlet.cs` | 4 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterfaceSecurity.cs` | 3 | +| `src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterfaceProgress.cs` | 2 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: profiles and startup scripts run under the same policy and logging as user input; lockdown policy is applied before any user code runs and cannot be disabled by command-line switches; server modes require an explicit privilege and authenticate their peer; encoded commands are decoded and logged in clear text. + +## 15. Enterprise considerations + +Security first: §14. Observability: ETW startup/perf events (`Perftrack_ConsoleStartup`), profile-load time, transcript options in policy, exit codes distinguishing usage (64), init failure (70) and Ctrl+Break (149). Supportability: stable command-line switches with documented prefixes; `-SettingsFile`. Testability: strong Pester coverage of host behavior; the parser has xUnit tests. Resource limits: none. Missing: a machine-readable startup diagnostic output, signed profiles by default, per-switch policy control other than file-only entry. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Host/Base-Directory.Tests.ps1` | 2 | 2 | 2 | +| `test/powershell/Host/ConsoleHost.Tests.ps1` | 9 | 22 | 89 | +| `test/powershell/Host/Logging.Tests.ps1` | 3 | 0 | 9 | +| `test/powershell/Host/PSVersionTable.Tests.ps1` | 1 | 1 | 15 | +| `test/powershell/Host/Startup.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/Host/TabCompletion/TabCompletion.Tests.ps1` | 4 | 25 | 249 | +| `test/powershell/Modules/PSReadLine/PSReadLine.Tests.ps1` | 1 | 0 | 8 | +| **total (7 files)** | 21 | 50 | 373 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_CommandLineParser.cs` | 55 | 52 | `PSCommandLineParserTests`, `TestDataSettingsFile`, `TestDataLastFile` | + +Matching is by path keyword (`APPROX`): `Host/ConsoleHost`, `Host/Startup`, `Host/Logging`, `Host/Base-Directory`, `Host/PSVersionTable`, `PSReadLine`, `Host/TabCompletion/TabCompletion`, `test_CommandLineParser`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: line editing is an external module loaded into the session, so the host stays small and users can replace it. +- `FACT-SRC`: exit code derives from `$?` of the last statement unless `exit` is used; scripts rely on this for CI. +- `INFERENCE`: appending `Out-Default` in the host (not in the engine) keeps the engine free of display policy; the engine appends it only for transcription. +- `FACT-SRC`: profiles are dot-sourced as separate pipelines so one failing profile does not stop the rest. +- `FACT-SRC`: server modes live in the same executable, selected by switches. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-HOSTCON-01** [REPLICATE] (area: NLD-RUN) Keep the exit-code rule and the specific codes (0, 1, 64, 70, 149) so automation behaves the same. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoRunspaceLoop@L1643-L1706. +- **LSN-HOSTCON-02** [REPLICATE] (area: NLD-RUN) Four-profile order and per-profile fault isolation; `-NoProfile` skips all. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoRunspaceInitialization@L1882-L2159. +- **LSN-HOSTCON-03** [ADAPT] (area: NLD-RUN) Keep line editing external, but define a small editor interface (read line with completion and history) instead of loading a module by name. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoCreateRunspace@L1766-L1866. +- **LSN-HOSTCON-04** [ADAPT] (area: NLD-RUN) Separate the executable into a thin launcher and a library that exposes `Start(args)` and `Start(initialSessionState, args)` like `ConsoleShell`. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs::ConsoleShell@L15-L80. +- **LSN-HOSTCON-05** [REDESIGN] (area: NLD-SEC) Apply policy (lockdown, language mode, allowed switches) from a signed policy source before parsing switches that widen access, and refuse to start server modes without an authenticated transport. DEV: DEV-HOST-200. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.Start@L100-L344. +- **LSN-HOSTCON-06** [AVOID] (area: NLD-RUN) Do not carry the Windows-only pieces (`ConsoleControl` Win32 calls, jump list, `-windowstyle`, `-sta` by default, Hyper-V socket modes) into the first version. DEV: DEV-HOST-201. Evidence: src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleControl.cs::ConsoleControl@L47-L3188. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-060): `ConsoleHostUserInterface` (prompt, progress, VT handling, `ReadLineWithTabCompletion`), `ConsoleHostRawUserInterface`, `ConsoleControl`, progress rendering, `CommandLineParameterParser` bodies (value handling of `-command -`, `-file -`, `-encodedcommand`, `-login`), `Executor` async details, `PushRunspace`/`PopRunspace`, debugger stop handling (`OnExecutionSuspended`), `ConsoleHostTranscript`, the native launcher. +- `UNVERIFIED` (VB-061): the PSReadLine module (outside this repository) and the `prompt`/feedback evaluation. + +Verification: `ManagedEntrance.cs` and `ConsoleShell.cs` (whole), `ConsoleHost.cs` lines 100–344, 484–560, 1576–1710, 1748–1881, 1882–2200, 2600–2830, `CommandLineParameterParser.cs` switch names (grep), `Executor.cs` lines 150–260 and 330–400 were read. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/04-hosting/host-abstraction.md b/docs-out/04-hosting/host-abstraction.md new file mode 100644 index 0000000..db58e32 --- /dev/null +++ b/docs-out/04-hosting/host-abstraction.md @@ -0,0 +1,182 @@ +--- +id: HOST-ABSTRACTION-001 +title: Host abstraction +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/hostifaces/MshHost.cs, src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs, src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs, src/System.Management.Automation/engine/hostifaces/InternalHost.cs, src/System.Management.Automation/engine/hostifaces/DefaultHost.cs, src/System.Management.Automation/engine/InitialSessionState.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/hostifaces/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Host abstraction + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. The engine side of host calls is in [runspace-and-pipeline](../03-runtime/runspace-and-pipeline.md) and [formatting-and-output](../03-runtime/formatting-and-output.md); the concrete console host is [console-host](console-host.md); embedding is [sdk](sdk.md). + +## 1. Purpose and scope [SPEC] + +The engine never touches a console. Everything user-facing goes through a *host*: an object derived from `PSHost` that supplies identity, culture, a user interface (`PSHostUserInterface`) and optionally a raw terminal interface (`PSHostRawUserInterface`), and that receives lifecycle notifications. This document describes the minimal contract a new host must implement, the engine-side wrapper (`InternalHost`) and the runspace configuration object (`InitialSessionState`) that a host supplies when it creates a runspace. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/hostifaces/MshHost.cs::PSHost@L37-L287 | abstract class | the host contract | B | +| src/System.Management.Automation/engine/hostifaces/MshHost.cs::IHostSupportsInteractiveSession@L294-L327 | interface | push/pop a runspace (interactive remoting) | B | +| src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs::PSHostUserInterface@L26-L1187 | abstract class | text I/O, prompts, progress, transcription hooks | B | +| src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs::PSHostRawUserInterface@L1165-L1914 | abstract class | keyboard, window, buffer | B | +| src/System.Management.Automation/engine/hostifaces/InternalHost.cs::InternalHost@L27-L580 | class | engine-side wrapper that can swap the external host | B | +| src/System.Management.Automation/engine/hostifaces/InternalHostUserInterface.cs::InternalHostUserInterface@L15-L1032 | class | wrapper of the UI with informational-message buffers | B | +| src/System.Management.Automation/engine/hostifaces/DefaultHost.cs::DefaultHost@L17-L134 | class | a minimal host used when none is given | B | +| src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState@L1254-L5025 | class | what a runspace starts with | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the host contract and its engine wrapper, nested prompt bookkeeping, the runspace-creation configuration. Does not own: console behavior (T42), line editing (an external module, see [console-host](console-host.md)), host calls over remoting (Phase 4 remoting), formatting. + +## 4. Key types and contracts [IMPL] + +- **`PSHost` — abstract members**: `Name`, `Version`, `InstanceId` (a GUID), `UI` (`PSHostUserInterface`), `CurrentCulture`, `CurrentUICulture`, `SetShouldExit(int exitCode)`, `EnterNestedPrompt()`, `ExitNestedPrompt()`, `NotifyBeginApplication()`, `NotifyEndApplication()`. Virtual with defaults: `PrivateData` (null), `DebuggerEnabled` (false; the setter throws `PSNotImplementedException`). The interface `IHostSupportsInteractiveSession` adds `PushRunspace`, `PopRunspace`, `IsRunspacePushed`, `Runspace` (src/System.Management.Automation/engine/hostifaces/MshHost.cs::PSHost@L37-L287). +- **`PSHostUserInterface` — abstract**: `RawUI`, `ReadLine`, `ReadLineAsSecureString`, `Write(string)`, `Write(fg, bg, string)`, `WriteLine(string)`, `WriteErrorLine`, `WriteDebugLine`, `WriteVerboseLine`, `WriteWarningLine`, `WriteProgress(sourceId, ProgressRecord)`, `Prompt(caption, message, fields)`, `PromptForCredential` (two overloads), `PromptForChoice(caption, message, choices, default)`. Virtual: `SupportsVirtualTerminal` (false), `WriteLine()`, `WriteLine(fg, bg, string)`, `WriteInformation(InformationRecord)` (empty). The transcription helpers (`TranscribeCommand`, `TranscribeResult`, …) are internal members of this base class (see [errors-and-diagnostics](../03-runtime/errors-and-diagnostics.md) §4). +- **`PSHostRawUserInterface` — abstract** properties: `ForegroundColor`, `BackgroundColor`, `CursorPosition`, `WindowPosition`, `CursorSize`, `BufferSize`, `WindowSize`, `MaxWindowSize`, `MaxPhysicalWindowSize`, `KeyAvailable`, `WindowTitle`; methods `ReadKey(options)`, `FlushInputBuffer`, `SetBufferContents` (two overloads), `GetBufferContents(rectangle)`, `ScrollBufferContents`. A host that is not a terminal can throw `NotImplementedException` for these; the engine tolerates it where it probes (screen scraping support is detected by trying to read the buffer, see [command-discovery-and-invocation](../03-runtime/command-discovery-and-invocation.md) §7). +- **Engine wrapper** (`InternalHost`): holds the external host reference with `SetHostRef(psHost)`, `RevertHostRef()`, `IsHostRefSet`, `ExternalHost`; pipelines can run with a different host for the duration of one invocation (`InvocationSettings.Host`: `LocalPipeline.InvokeThreadProc` sets it before the pipeline runs and reverts it in `finally`). `NestedPromptCount` counts nested prompts. +- **Nested prompt** (`InternalHost.EnterNestedPrompt(callingCommand)`): refuses to nest while the running pipeline is the internal "pulse" pipeline; increments the count and sets `$NestedPromptLevel`; saves context data (current variables for the executing command and bound parameters, runspace availability, language mode); sets the "currently executing command" variable to the calling command wrapped with `CommandInfo` and `StackTrace` note properties; resets debug trace settings and the shell-function error pipe; **if the runspace has ever used Constrained Language mode, the nested prompt runs in Constrained Language**; marks the runspace `AvailableForNestedCommand` (lines 230-330 read; the call into the external host that follows is `UNVERIFIED`) (src/System.Management.Automation/engine/hostifaces/InternalHost.cs::InternalHost.EnterNestedPrompt@L230-L361). The restore on exit was not read (`UNVERIFIED`, VB-058). +- **`InitialSessionState`** — what a host passes to `RunspaceFactory.CreateRunspace(host, iss)`: factories `Create()` (empty: no commands, no variables; `LanguageMode` property default is `NoLanguage`), `CreateDefault()` (built-in variables, functions and aliases, all applications and scripts allowed (`*`), the engine snap-ins' cmdlets and providers; **FullLanguage**; the standard authorization manager), `CreateDefault2()` (same but imports only the core snap-in via `ImportCorePSSnapIn`; FullLanguage; the standard authorization manager), `CreateRestricted(SessionCapabilities.RemoteServer)` (empty plus built-in functions and aliases, `NoLanguage`, `ThrowOnRunspaceOpenError = true`, no full language in the debugger), `CreateFromSessionConfigurationFile(path[, roleVerifier, validateFile])`; `Clone()`. Properties: `LanguageMode`, `ApartmentState`, `ThreadOptions`, `ThrowOnRunspaceOpenError`, `UseFullLanguageModeInDebugger`, `TranscriptDirectory`, `AuthorizationManager`, `DisableFormatUpdates`; collections `Commands`, `Providers`, `Variables`, `Assemblies`, `Types`, `Formats`, `StartupScripts`, `Modules` (via `ImportPSModule`, `ImportPSModulesFromPath`, `ImportPSSnapIn`) (src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.CreateDefault@L1524-L1583, InitialSessionState.CreateRestricted@L1378-L1387, InitialSessionState.CreateDefault2@L1591-L1606). + +## 5. Data model + +A host is a plain object graph: identity (name, version, instance id), culture pair, UI object, optional private data (a `PSObject` of host-specific settings that the engine exposes as `$Host.PrivateData`). `InternalHost` adds the external host reference, a stack of saved contexts for nested prompts and the informational-message buffers (`InternalHostUserInterface.SetInformationalMessageBuffers`, set by the pipeline for the duration of a run). `InitialSessionState` entries (`SessionStateCommandEntry`, `…ProviderEntry`, `…VariableEntry`, …) carry a *visibility* (public/private), which restricted sessions use to hide commands. + +## 6. Control flow + +```mermaid +sequenceDiagram + participant H as Host + participant RF as RunspaceFactory + participant R as Runspace + participant E as Engine + participant UI as PSHostUserInterface + H->>RF: CreateRunspace(host, InitialSessionState) + RF->>R: Open (engine created, session state bound) + H->>R: pipeline Invoke + E->>UI: Write, WriteErrorLine, WriteProgress, Prompt, ReadLine + E->>H: NotifyBeginApplication / NotifyEndApplication around native programs + E->>H: EnterNestedPrompt / ExitNestedPrompt + E->>H: SetShouldExit(code) +``` + +Ordered: the host creates the runspace with its `PSHost` and an `InitialSessionState`; the engine wraps the host in `InternalHost`; commands call `$Host.UI.*`, `Write-Host`, `Read-Host`, `Out-Default` (through `ConsoleLineOutput` for the console host) which end in the host UI methods; native programs bracket their run with the begin/end notifications; `exit` calls `SetShouldExit`. + +## 7. Algorithms + +Not applicable beyond §4 (nested prompt state save/restore). + +## 8. Observable behavior [SPEC] + +- `$Host` is the (wrapped) host: `Name`, `Version`, `InstanceId`, `UI`, `CurrentCulture`, `PrivateData`, `EnterNestedPrompt()`, `ExitNestedPrompt()`. +- `Write-Host` writes to `UI.Write*` with colors; `Read-Host` calls `ReadLine`/`ReadLineAsSecureString`; prompting cmdlets call `Prompt`, `PromptForChoice`, `PromptForCredential`; `Write-Progress` calls `WriteProgress`. +- `exit` ends the session through `SetShouldExit`; a nested prompt (`$NestedPromptLevel` > 0) is left with `exit`. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +A host that throws `NotImplementedException`/`PSNotImplementedException` for optional members is tolerated in probing code (`InternalHost` catches `PSNotImplementedException` when reading the runspace); entering a nested prompt from the pulse pipeline is an `InvalidOperationException`; the nested-prompt counter going negative is an internal error (`EnterExitNestedPromptOutOfSync`); `DebuggerEnabled` set on a host that does not support it throws. + +## 10. Concurrency and threading + +Host methods are called on the pipeline thread (or on the `Out-*` consumer thread); a host used by several runspaces must be thread-safe. The external host reference is swapped under the pipeline thread (`LocalPipeline.InvokeThreadProc`); the console host protects its state with a global lock (see [console-host](console-host.md)). `SetShouldExit` can be called from engine threads. + +## 11. Extension points and customization hooks + +Subclass `PSHost` (and the UI classes); supply `PrivateData`; implement `IHostSupportsInteractiveSession` for interactive remoting; implement `IHostSupportsInteractiveSession`-style push/pop; `InitialSessionState` entry lists; `InvocationSettings.Host` per call; `PSHost.DebuggerEnabled`. Hosts that want colored/ANSI output declare `SupportsVirtualTerminal`. + +## 12. Dependencies + +Inbound: every cmdlet that writes or prompts, `Out-Default`, transcription, the progress pipeline, the debugger (`DebuggerStop` handling), remoting (host calls are forwarded). Outbound: culture settings, `ProgressRecord`, `FieldDescription`, `ChoiceDescription`, `PSCredential`, `SecureString`. + +## 13. Performance notes + +Host calls are synchronous; a slow host slows the pipeline. Output through `Write-Host` and `Out-Default` is line oriented; large outputs benefit from a host that buffers writes. `InitialSessionState.CreateDefault2` avoids loading all engine snap-ins eagerly, which shortens start-up. + +## 14. Security considerations + +The host is a trust boundary in both directions: the engine trusts the host to display messages faithfully and to implement prompts (consent for `ShouldContinue`, credentials), and the host trusts the engine not to flood or spoof its UI. Enforcement and policy seen: nested prompts inherit Constrained Language once it has been used in the runspace; `InitialSessionState` is where language mode, visibility, authorization manager and `ThrowOnRunspaceOpenError` are fixed for a session (`CreateRestricted` shows the minimal remote-server baseline); the UI carries the transcription hooks that record input and output. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| logging | 94 | +| language-mode | 38 | +| jea | 17 | +| execution-policy | 8 | +| securestring | 7 | +| amsi | 1 | +| app-control | 1 | +| constrained-types | 1 | +| **total** | 167 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs` | 91 | +| `src/System.Management.Automation/engine/InitialSessionState.cs` | 64 | +| `src/System.Management.Automation/engine/hostifaces/InternalHost.cs` | 6 | +| `src/System.Management.Automation/engine/hostifaces/InternalHostUserInterface.cs` | 6 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: prompts for consent and credentials are rendered by the host, never forgeable by script output; a restricted `InitialSessionState` is the only way a session starts restricted and cannot be widened from inside; transcription and logging hooks live below the host so a custom host cannot disable them. + +## 15. Enterprise considerations + +Security first: §14. Observability: transcription, host `InstanceId`, ETW events around startup (console host). Supportability: a tiny required contract (11 abstract members on `PSHost`) makes hosts easy to write; the UI contract is larger. Testability: `DefaultHost` and test hosts allow headless engine tests. Missing: a structured (non-text) UI channel, a capability negotiation (the engine probes by exceptions), localizable consent prompts with policy. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Host/HostUtilities.Tests.ps1` | 5 | 0 | 7 | +| `test/powershell/Modules/Microsoft.PowerShell.Core/Out-Host.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Read-Host.Tests.ps1` | 1 | 0 | 7 | +| `test/powershell/Modules/Microsoft.PowerShell.Utility/Write-Host.Tests.ps1` | 3 | 0 | 3 | +| `test/powershell/engine/Api/InitialSessionState.Tests.ps1` | 2 | 2 | 8 | +| **total (5 files)** | 12 | 2 | 26 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_Runspace.cs` | 3 | 0 | `RunspaceTests` | + +Matching is by path keyword (`APPROX`): `Host/HostUtilities`, `Utility/Read-Host`, `Utility/Write-Host`, `Core/Out-Host`, `engine/Api/InitialSessionState`, `test_Runspace`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: a small abstract core with large optional surfaces (raw UI) keeps simple hosts simple. +- `FACT-SRC`: the engine holds the host through a wrapper, so a pipeline can temporarily use a different host. +- `INFERENCE`: capability probing by catching `NotImplemented` exceptions keeps the contract stable across versions. +- `FACT-SRC`: nested prompts carry the strictest language mode seen in the runspace. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-HOSTABS-01** [REPLICATE] (area: NLD-RUN) Keep the three-level host contract (host, UI, raw UI) and the lifecycle notifications; host authors depend on them. Evidence: src/System.Management.Automation/engine/hostifaces/MshHost.cs::PSHost@L37-L287. +- **LSN-HOSTABS-02** [ADAPT] (area: NLD-RUN) Replace exception-based capability probing with explicit capability flags on the host (supports raw UI, VT, interactive session, debugger). Evidence: src/System.Management.Automation/engine/hostifaces/MshHost.cs::PSHost@L37-L287. +- **LSN-HOSTABS-03** [REPLICATE] (area: NLD-SEC) Nested prompts and debugger prompts run at the strictest language mode seen. Evidence: src/System.Management.Automation/engine/hostifaces/InternalHost.cs::InternalHost.EnterNestedPrompt@L230-L361. +- **LSN-HOSTABS-04** [ADAPT] (area: NLD-RUN) Provide a headless default host and typed prompt results as part of the SDK so embedders do not subclass the full UI contract. Evidence: src/System.Management.Automation/engine/hostifaces/DefaultHost.cs::DefaultHost@L17-L134. +- **LSN-HOSTABS-05** [REPLICATE] (area: NLD-SEC) A restricted-session baseline factory (empty, no language, fail on open error). Evidence: src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.CreateRestricted@L1378-L1387. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-058): `InternalHost` exit-nested-prompt path and remaining members, `InternalHostUserInterface` buffering, `PSHostUserInterface` transcription code beyond the cited hooks, `HostUtilities` (profile paths, prompt helpers), `DefaultHost` behavior. +- `UNVERIFIED` (VB-059): `InitialSessionState` entry classes, `Bind`, `CreateFromSessionConfigurationFile` (JEA, see [jea](../05-remoting/jea.md)), snap-in handling, `Clone` semantics. + +Verification: `MshHost.cs` (abstract/virtual members by grep, lines 222–327), `MshHostUserInterface.cs` and `MshHostRawUserInterface.cs` (member lists by grep), `InternalHost.cs` lines 225–330, `InitialSessionState.cs` lines 1310–1387, 1498–1606, 1749–1850 were read; everything else is from signatures. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/04-hosting/sdk.md b/docs-out/04-hosting/sdk.md new file mode 100644 index 0000000..bd494f9 --- /dev/null +++ b/docs-out/04-hosting/sdk.md @@ -0,0 +1,177 @@ +--- +id: HOST-SDK-001 +title: SDK and embedding +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/hostifaces/PowerShell.cs, src/System.Management.Automation/engine/hostifaces/ConnectionFactory.cs, src/System.Management.Automation/engine/hostifaces/Connection.cs, src/System.Management.Automation/engine/MshCmdlet.cs, src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj, docs/dev-process/breaking-change-contract.md, docs/host-powershell/README.md] +mirror_cards: [src-mirror/src/Microsoft.PowerShell.SDK/_project.md, src-mirror/src/System.Management.Automation/_project.md] +generated_from: [data/types.json, data/projects.json, data/tests.json, data/enforcement-points.json] +--- + +# SDK and embedding + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. Engine behavior behind the API: [runspace-and-pipeline](../03-runtime/runspace-and-pipeline.md) (states, threads, stop). Host contract: [host-abstraction](host-abstraction.md). + +## 1. Purpose and scope [SPEC] + +How a .NET application embeds the engine: create a runspace (or a pool), build commands with the fluent `PowerShell` API, invoke them synchronously or asynchronously, read results and streams, and configure the session. The document also covers the NuGet packaging model and the API stability policy. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PowerShell@L575-L5954 | sealed class | fluent API over a runspace or pool | B | +| src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PSInvocationSettings@L313-L415 | sealed class | per-invocation settings | B | +| src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PSDataStreams@L5959-L6142 | sealed class | the six non-output streams of an invocation | B | +| src/System.Management.Automation/engine/hostifaces/ConnectionFactory.cs::RunspaceFactory@L17-L627 | static class | runspace and pool factories | B | +| src/System.Management.Automation/engine/hostifaces/Connection.cs::Runspace@L450-L1665 | abstract class | a runspace (open, close, create pipeline, state events) | B | +| src/System.Management.Automation/engine/hostifaces/Connection.cs::SessionStateProxy@L1671-L1894 | class | variable, language-mode and intrinsics access from outside a pipeline | B | +| src/System.Management.Automation/engine/MshCmdlet.cs::CommandInvocationIntrinsics@L238-L915 | class | `InvokeCommand` helpers (expand string, get command, lookup actions) | B | +| src/System.Management.Automation/engine/hostifaces/PSDataCollection.cs | class | thread-safe observable collection for input and output | B | +| src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj | project | metapackage | C | +| docs/dev-process/breaking-change-contract.md, docs/host-powershell/README.md | docs | stability policy and a hosting guide with samples | C | + +## 3. Responsibilities and non-responsibilities + +Owns: the embedding surface and its packaging. Does not own: engine internals, the console host (T42), remoting classes (Phase 4 remoting), the `Microsoft.PowerShell.SDK` dependency pins' upstream content. + +## 4. Key types and contracts [IMPL] + +- **Create**: `PowerShell.Create()` (a new default runspace), `Create(RunspaceMode)` (`CurrentRunspace` = 0 or `NewRunspace` = 1), `Create(InitialSessionState)`, `Create(Runspace)`; `CreateNestedPowerShell()` for use inside a running pipeline. `RunspaceFactory.CreateRunspace()` (default state), `(PSHost)`, `(InitialSessionState)`, `(PSHost, InitialSessionState)`, `(RunspaceConnectionInfo[, host, typeTable, applicationArguments, name])` for remote runspaces, `CreateOutOfProcessRunspace(typeTable[, PowerShellProcessInstance])`; `CreateRunspacePool(min, max, …)` with overloads for state, host, connection info (src/System.Management.Automation/engine/hostifaces/ConnectionFactory.cs::RunspaceFactory@L17-L627). A runspace must be opened (`Open`) before pipelines run; a `PowerShell` created over a runspace requires state `Opened` at invoke time, else `InvalidRunspaceStateException` (see runspace document §10). +- **Build**: `AddCommand(name[, useLocalScope])`, `AddCommand(CommandInfo)`, `AddScript(text[, useLocalScope])`, `AddParameter(name[, value])`, `AddStatement()` (starts a new statement; mirrors `;`), `Commands` (a `PSCommand` that can be cloned and reused). +- **Invoke**: `Invoke()`, `Invoke(input)`, `Invoke(input, settings)`, typed `Invoke` variants and `Invoke(input, output, settings)`; asynchronous `BeginInvoke…`/`EndInvoke`, `InvokeAsync…` (task-based), `Stop()`, `BeginStop(callback, state)`; `Dispose`. State is `PSInvocationState` (NotStarted, Running, Stopping, Stopped, Completed, Failed, Disconnected); a `PowerShell` object is single-use per invocation but its commands can be cleared and reused (`UNVERIFIED` exact reuse rule). `PSInvocationSettings`: `ApartmentState`, `Host` (per-call host), `RemoteStreamOptions`, `AddToHistory`, `ErrorActionPreference`, `FlowImpersonationPolicy`, `ExposeFlowControlExceptions` (src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PSInvocationSettings@L313-L415). +- **Results and streams**: output comes back as `Collection` (or typed); input is supplied as a `PSDataCollection` that can be filled while the command runs; `Streams` (`PSDataStreams`) exposes `Error`, `Warning`, `Verbose`, `Debug`, `Progress`, `Information` collections, each raising `DataAdded` events; `HadErrors` reports whether any non-terminating error occurred; terminating errors surface as exceptions from `Invoke` (for example `CmdletInvocationException`) or as the failed state with a reason in async use. +- **Session access without a pipeline**: `Runspace.SessionStateProxy` (`SetVariable`, `GetVariable`, `LanguageMode`, `Applications`, `Scripts`, `Drive`, `Path`, `Provider`, `PSVariable`, `InvokeCommand`, `InvokeProvider`, `Module`); it is rejected while a pipeline is running except while the internal pulse pipeline runs (see runspace document §10). `CommandInvocationIntrinsics` (`$ExecutionContext.InvokeCommand`): `ExpandString`, `GetCommand(name, types[, args])`, `GetCmdlet`, `GetCmdletByTypeName`, `GetCmdlets`, the lookup actions (`PreCommandLookupAction`, `PostCommandLookupAction`, `CommandNotFoundAction`, `LocationChangedAction`) and `HasErrors`. +- **Packaging**: `Microsoft.PowerShell.SDK` is a **metapackage** (`IncludeBuildOutput` false) that references the projects `System.Management.Automation`, `Microsoft.PowerShell.Commands.Utility`, `…Commands.Management`, `…Security` and `…ConsoleHost`, and pins the versions of non-direct .NET packages (for example `System.Data.*`, `System.DirectoryServices*`, `System.Diagnostics.*`, native `System.IO.Ports` runtime packages for each RID) so applications get a consistent runtime set; target framework `net10.0` (`PowerShell.Common.props`). The engine project itself depends on `Newtonsoft.Json`, `Microsoft.ApplicationInsights`, `Microsoft.Management.Infrastructure` (Windows/CIM), `Microsoft.PowerShell.Native` and several `System.*` packages. The hosting guide says runtime assemblies for Windows, Linux and macOS are published as NuGet packages and a sample application builds per runtime identifier (`docs/host-powershell/README.md`). +- **Stability policy** (`docs/dev-process/breaking-change-contract.md`): four buckets — *public contract* (renaming or removing a public type, member or cmdlet parameter, narrowing accepted values, changing a public constant or enum value, changing existing behavior for a given input, changing a protocol without a version bump are unacceptable; adding types, members and cmdlets, changing error behavior into new functionality, and protocol changes with a version increment are acceptable), *reasonable grey area* (timing/order of events, parser strictness: needs judgement, preview and possibly an RFC), *unlikely grey area* and *clearly non-public*. Preview features may change. + +## 5. Data model + +`PowerShell` holds: a `PSCommand` (list of command lists = statements), a connection (runspace or pool), invocation state info, `Streams`, `HadErrors`, instance id, an `IsNested` flag, and async result objects. `PSDataCollection` is a growable list with a completion state, blocking enumeration and `DataAdded`, `DataAdding`, `Completed` events; it is the unit for streaming input and output. + +## 6. Control flow + +```mermaid +sequenceDiagram + participant App as Application + participant PS as PowerShell + participant RS as Runspace or Pool + participant PL as Pipeline + App->>PS: Create, AddScript or AddCommand, AddParameter + App->>PS: Invoke or BeginInvoke + PS->>RS: get runspace (pool blocks until one is free) + RS->>PL: create pipeline from PSCommand + PL-->>PS: output collection, stream events, state events + PS-->>App: results, Streams, HadErrors or exception + App->>PS: Stop (optional), Dispose +``` + +## 7. Algorithms + +Not applicable beyond the invoke path described in the runspace document (runspace acquisition, pipeline construction, state transitions). + +## 8. Observable behavior [SPEC] + +- A snippet such as `PowerShell.Create().AddScript("Get-Verb -Verb get").Invoke()` returns the command output as `PSObject`s; `AddCommand("Test-Path").AddParameter("Path", p).Invoke()` returns typed output (these shapes are exercised by the hosting tests). +- Errors that are non-terminating appear in `Streams.Error` and set `HadErrors`; terminating errors throw from `Invoke`. +- Several `PowerShell` objects can share a `RunspacePool`; one runspace runs one pipeline at a time. +- Some cmdlets are restricted in hosted contexts: the xUnit test asserts that `Start-Job` in a hosted scenario throws `CmdletInvocationException` (`UNVERIFIED` reason: background jobs need the PowerShell executable). +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`InvalidRunspaceStateException` / `InvalidPowerShellStateException` for wrong states; `ObjectDisposedException` after dispose; `ConcurrentInvokeNotAllowed` when a runspace already runs a pipeline; async exceptions are delivered through the state change (`Failed`) and `EndInvoke`; `PipelineStoppedException` for stopped pipelines; the host-side exceptions from `PSHost` implementations propagate as cmdlet errors. + +## 10. Concurrency and threading + +A `PowerShell` instance is not safe for concurrent use, except `Stop`/`BeginStop` and reading streams; collections are thread-safe. The thread that runs script is chosen by `PSThreadOptions` (see runspace document); pool runspaces use `ReuseThread` unless set otherwise. Events (`InvocationStateChanged`, `DataAdded`) fire on engine threads. + +## 11. Extension points and customization hooks + +Custom `PSHost`; `InitialSessionState` (cmdlets, providers, functions, variables, modules, language mode, authorization manager); `PSInvocationSettings`; `Runspace.Events`; `SessionStateProxy`; `PSCommand` reuse; custom cmdlets compiled into the app and registered through `SessionStateCmdletEntry`; `CommandInvocationIntrinsics` lookup actions for command resolution policy; the metapackage for dependency alignment. + +## 12. Dependencies + +Inbound: embedding applications, the console host, remoting servers, test harnesses. Outbound: the engine runtime ([runspace-and-pipeline](../03-runtime/runspace-and-pipeline.md)), host contract, .NET runtime packages listed in the metapackage. + +## 13. Performance notes + +Creating a runspace is expensive (engine init, session state binding); applications should reuse runspaces or pools; `InitialSessionState.CreateDefault2` is cheaper than `CreateDefault`; reusing a `PSCommand`/script avoids reparsing is `UNVERIFIED`; async APIs avoid blocking threads; pools bound concurrency. + +## 14. Security considerations + +An embedder gets full control of the engine in its process: whatever the host process can do, scripts can do unless the `InitialSessionState` restricts it. The knobs are language mode, command visibility, `Applications`/`Scripts` lists, `AuthorizationManager`, restricted factories (`CreateRestricted`), constrained endpoints (JEA) and per-call `ErrorActionPreference`. Input objects and `AddParameter` values are passed by reference to script (no copying), so untrusted .NET objects with powerful members can reach script. `AddScript` runs arbitrary text; building scripts from untrusted strings is injection-prone, `AddCommand` + `AddParameter` is the safe form. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| language-mode | 3 | +| **total** | 3 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/hostifaces/Connection.cs` | 3 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: the embedding API defaults to a restricted session (explicit opt-in to full language); parameters and input values are typed data, never concatenated into script text by the API; logging and audit hooks cannot be removed by the embedder without a privileged policy; the API surface is versioned with a published compatibility contract. + +## 15. Enterprise considerations + +Security first: §14. Observability: state events, streams, host transcription, ETW. Supportability: the stability policy is explicit and public; breaking changes need justification; protocols carry versions. Testability: the API is exercised by xUnit and Pester tests that host the engine. Resource limits: pool min/max, no per-invocation memory/time budget. Missing: a built-in per-invocation timeout/cancellation token API (only `Stop`), structured result types beyond `PSObject`, source-generated or reference assemblies listed in this repository. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/SDK/Breakpoint.Tests.ps1` | 1 | 5 | 35 | +| `test/powershell/SDK/Json.Tests.ps1` | 1 | 0 | 6 | +| `test/powershell/SDK/PSDebugging.Tests.ps1` | 2 | 2 | 12 | +| `test/powershell/engine/Api/BasicEngine.Tests.ps1` | 3 | 2 | 9 | +| `test/powershell/engine/Api/PSCommand.Tests.ps1` | 1 | 2 | 4 | +| `test/powershell/engine/Api/Serialization.Tests.ps1` | 1 | 0 | 7 | +| `test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1` | 1 | 4 | 9 | +| **total (7 files)** | 10 | 15 | 82 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_PSObject.cs` | 8 | 0 | `PSObjectTests` | +| `test/xUnit/csharp/test_PowerShellAPI.cs` | 1 | 0 | `PowerShellHostingScenario` | +| `test/xUnit/csharp/test_SessionState.cs` | 0 | 0 | `SessionStateTests` | + +Matching is by path keyword (`APPROX`): `SDK/`, `engine/Api/PSCommand`, `engine/Api/BasicEngine`, `engine/Api/TaskBasedAsyncPowerShellAPI`, `engine/Api/Serialization`, `test_PowerShellAPI`, `test_SessionState`, `test_PSObject`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: results are `PSObject` wrappers so scripts and C# see the same member view (ETS); typed overloads convert on the way out. +- `FACT-SRC`: the metapackage exists to keep a consistent set of transitive packages rather than expressing them as dependencies of the engine assembly. +- `INFERENCE`: a fluent builder over `PSCommand` keeps one object model for local, remote and pool invocation. +- `FACT-SRC`: stability is governed by a written contract that treats behavior as part of the public surface. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-HOSTSDK-01** [REPLICATE] (area: NLD-RUN) A fluent command builder (command, script, parameter, statement) with sync, async and streaming-input invocation. Evidence: src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PowerShell@L575-L5954. +- **LSN-HOSTSDK-02** [ADAPT] (area: NLD-RUN) Add cancellation tokens and per-invocation time and memory budgets to the invoke API (the original has `Stop` only). Evidence: src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PSInvocationSettings@L313-L415. +- **LSN-HOSTSDK-03** [REDESIGN] (area: NLD-SEC) Make the default embedded session restricted and require an explicit opt-in to full language and unrestricted command visibility. DEV: DEV-SDK-205. Evidence: src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.CreateRestricted@L1378-L1387. +- **LSN-HOSTSDK-04** [REPLICATE] (area: NLD-RUN) Publish a written compatibility contract that classifies changes and requires a protocol version bump for wire changes. Evidence: docs/dev-process/breaking-change-contract.md@L1-L60. +- **LSN-HOSTSDK-05** [ADAPT] (area: NLD-RUN) Ship a metapackage that pins transitive dependencies, but generate it from the build graph and publish reference assemblies for the public surface. Evidence: src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj@L1-L40. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-062): `PowerShell` reuse rules after completion, `PSCommand` semantics, `PSDataCollection` internals, `PSDataStreams`, `HadErrors` definition, `RunspacePool` public API details, `Connect`/disconnect APIs, `Runspace` events. +- `UNVERIFIED` (VB-063): `SessionStateProxy` implementation and its concurrency checks beyond those read for the runspace document; the sample application and per-RID publishing; whether reference assemblies are produced. + +Verification: member lists of `PowerShell`, `PSInvocationSettings`, `RunspaceFactory`, `SessionStateProxy` and `CommandInvocationIntrinsics` were taken from public-signature greps; `docs/dev-process/breaking-change-contract.md` (first 60 lines) and `docs/host-powershell/README.md` (first 30 lines) and the SDK/SMA project files (selected lines) were read. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/05-remoting/jea.md b/docs-out/05-remoting/jea.md new file mode 100644 index 0000000..980b81a --- /dev/null +++ b/docs-out/05-remoting/jea.md @@ -0,0 +1,172 @@ +--- +id: REM-JEA-001 +title: Session configurations and constrained endpoints (JEA) +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: OUT-OF-SCOPE-DOCUMENTED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs, src/System.Management.Automation/engine/InitialSessionState.cs, src/System.Management.Automation/engine/remoting/commands/CustomShellCommands.cs, src/System.Management.Automation/engine/remoting/commands/NewPSSessionConfigurationFile.cs, src/System.Management.Automation/engine/remoting/fanin/PSSessionConfigurationData.cs, src/System.Management.Automation/engine/remoting/fanin/PSPrincipal.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/remoting/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Session configurations and constrained endpoints (JEA) + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `OUT-OF-SCOPE-DOCUMENTED`: remoting is not first-version scope, but the *declarative restricted-session mechanism* is the model for how a new shell should build locked-down sessions locally. Language modes and enforcement points: Phase 5 (`07-security/`). Session state basics: [execution-context-and-session-state](../03-runtime/execution-context-and-session-state.md); `InitialSessionState`: [host-abstraction](../04-hosting/host-abstraction.md). + +## 1. Purpose and scope [SPEC] + +A *session configuration* decides what a remote (or local, with `-ConfigurationFile`) runspace looks like when it starts. **Just Enough Administration (JEA)** uses configuration files to expose a narrow, audited command surface to non-administrators. Two file kinds: a **session configuration file** (`.pssc`) describing the endpoint (session type, language mode, visible commands, role mapping, run-as identity) and **role capability files** (`.psrc`) describing what a role may do. This document describes the file model, how the engine turns it into an `InitialSessionState`, how visibility restrictions are applied, and what the engine does *not* enforce itself. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::ConfigFileConstants | static class | the recognized keys and their validators | B | +| src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCUtils | static class | load, validate and merge configuration files; role definitions | B | +| src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration | class | builds an `InitialSessionState` from a configuration (`GetInitialSessionState(PSSenderInfo)`) | B | +| src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::PSSessionConfiguration | abstract public class | endpoint extensibility (`GetInitialSessionState`, size limits, private data) | B | +| src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.CreateFromSessionConfigurationFile@L1310-L1313 | method | entry used by `pwsh -ConfigurationFile` and the SDK | B | +| src/System.Management.Automation/engine/remoting/commands/CustomShellCommands.cs | cmdlets | `Register-PSSessionConfiguration` family (endpoint registration, SDDL, required groups) | B | +| src/System.Management.Automation/engine/remoting/commands/NewPSSessionConfigurationFile.cs | cmdlet | generates a `.pssc` | B | +| src/System.Management.Automation/engine/remoting/fanin/PSPrincipal.cs | classes | caller identity and `PSSenderInfo` | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the configuration file schema and validation, role merging, translation to an initial session state with command visibility, and the in-session checks that rely on visibility. Does not own: endpoint registration in the OS (WinRM/SSH), group-based authorization (SDDL / `RequiredGroups` are applied by the registration cmdlets and the OS), virtual account creation (Windows), event logging policy. + +## 4. Key types and contracts [IMPL] + +- **Session configuration file keys** (validated against a table `ConfigFileKeys`; unknown keys are rejected): `SchemaVersion`, `GUID`, `Author`, `Description`, `CompanyName`, `Copyright`, `PowerShellVersion`, `SessionType`, `LanguageMode`, `ExecutionPolicy`, `ModulesToImport`, `AssembliesToLoad`, `ScriptsToProcess`, `TypesToProcess`, `FormatsToProcess`, `EnvironmentVariables`, `VariableDefinitions`, `AliasDefinitions`, `FunctionDefinitions`, `VisibleAliases`, `VisibleCmdlets`, `VisibleFunctions`, `VisibleProviders`, `VisibleExternalCommands`, `RoleDefinitions`, `RoleCapabilities`, `RoleCapabilityFiles`, `RequiredGroups`, `RunAsVirtualAccount`, `RunAsVirtualAccountGroups`, `GroupManagedServiceAccount`, `MountUserDrive`, `UserDriveMaximumSize`, `TranscriptDirectory`, `EnforceInputParameterValidation` (src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs@L925-L1015). +- **`SessionType`**: `Empty` (an empty session state; core snap-in imported only when modules need it), `RestrictedRemoteServer` (= `InitialSessionState.CreateRestricted(SessionCapabilities.RemoteServer)`: empty, no language, built-in functions and aliases, throw on open errors), `Default` (= `CreateDefault2()`: full engine, full language); a missing value means `Default`. +- **Role capabilities**: `RoleDefinitions` maps a group/user name to customizations (`RoleCapabilities` names or `RoleCapabilityFiles` paths plus inline keys); a role verifier callback (caller's group membership, supplied by the caller of `GetInitialSessionState`) selects which role entries apply. A `.psrc` file is found by name: `RoleCapabilities = 'Module\Name'` or `'Name'` searches each `PSModulePath` entry for a module folder (any, or the named one) containing `RoleCapabilities\.psrc`; `RoleCapabilityFiles` must end in `.psrc` and exist. **Merging**: for every key, the existing value and the role's value are flattened into one array (values are *concatenated*, never overridden), so several roles' visible commands add up ([src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.MergeConfigHashIntoConfigHash@L1887-L1929]). +- **Visibility model**: when any of `VisibleCmdlets`, `VisibleFunctions`, `VisibleAliases`, `VisibleProviders`, `VisibleExternalCommands` is specified with non-default content, `iss.DefaultCommandVisibility` becomes **Private** and, for the default session type, every existing command entry is switched to private; then each name listed is made **Public** (`ProcessVisibleCommand`: for a bare name, all cmdlet and function entries of that name; for `Module\Name` the command from that module, importing the module name list as context). Private commands cannot be invoked by the remote caller (origin `Runspace`) but remain callable from inside the session by other commands (origin `Internal`) — this is what the command-resolution `ShouldRun` check implements (see [command-discovery-and-invocation](../03-runtime/command-discovery-and-invocation.md) §6). Visible providers are looked up in the provider list; visible external commands become the session's `Applications`/`Scripts` allow-list (`UNVERIFIED` mapping). +- **Parameter-level restriction**: an entry in `VisibleCmdlets`/`VisibleFunctions` may be a hashtable `@{ Name = 'Restart-Service'; Parameters = @{ Name='Name'; ValidateSet='Spooler','DnsClient' } , @{ Name = 'Force'; ValidatePattern = '…' } }`: each parameter entry needs a `Name`; the other keys (validation attribute kinds) collect sets of allowed values per command and parameter and are stored as `iss.CommandModifications`; a syntactically wrong entry throws `InvalidVisibleCommandKeyEntries` ([src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.ProcessCommandModification@L2527-L2612]). Whether and how the engine applies `CommandModifications` when proxying commands is `UNVERIFIED` (VB-070); `EnforceInputParameterValidation = true` is set on the session state at the end. +- **Other keys applied**: `ModulesToImport` (names or module specification hashtables; each name also feeds the module-name context for visible commands), `AssembliesToLoad`, `ScriptsToProcess` (run at session start), `TypesToProcess`/`FormatsToProcess`, `VariableDefinitions`, `AliasDefinitions` (a definition replaces an existing alias entry of the same name), `FunctionDefinitions` (script-block functions added to the session, which become the privileged "proxy" implementations), `EnvironmentVariables`, `LanguageMode` (validated enum), `ExecutionPolicy` (validated), `TranscriptDirectory`, `MountUserDrive`/`UserDriveMaximumSize` (a per-user temporary drive with a size cap). Role-definition validation checks types and extensions (`ValidateExtensions`, `ValidateAbsolutePaths`). +- **Endpoint extensibility**: `PSSessionConfiguration` (public abstract) lets an endpoint implement `GetInitialSessionState(PSSenderInfo)`, `GetMaximumReceivedObjectSize`, `GetMaximumReceivedDataSizePerCommand`, `GetApplicationPrivateData`; the WSMan plugin loads an endpoint's configuration type from registry-configured assembly/type or from the `.pssc` (`DISCPowerShellConfiguration`). + +## 5. Data model + +A configuration is a hashtable (`_configHash`) parsed from the `.pssc`/`.psrc` as restricted data (no code evaluation), validated key by key, with role customizations merged in as arrays. The result is an `InitialSessionState` (commands with visibility, providers, modules, variables, command modifications, language mode, execution policy, transcript directory, input validation flag). Caller identity (`PSSenderInfo`: user, connection info, application arguments) is available to server-side code. + +## 6. Control flow + +```mermaid +flowchart TD + F[.pssc file] --> L[load as restricted data and validate keys] + L --> R[merge role definitions selected by the role verifier] + R --> C[merge RoleCapabilities and RoleCapabilityFiles .psrc] + C --> T{SessionType} + T -->|Empty| E[empty InitialSessionState] + T -->|RestrictedRemoteServer| RS[CreateRestricted RemoteServer] + T -->|Default| D[CreateDefault2] + E --> V[apply modules, assemblies, definitions] + RS --> V + D --> V + V --> VIS[if Visible* given: default visibility Private, then make listed commands Public] + VIS --> M[command modifications, language mode, execution policy, input validation] + M --> S[runspace pool created from the session state] +``` + +## 7. Algorithms + +Visibility application (§4) and role merge (§4). Order inside `GetInitialSessionState`: determine session type; create the base state; decide whether visibility restrictions apply (`IsNonDefaultVisibilitySpecified` per category); set default visibility; apply visible providers; add assemblies; import modules; apply `VisibleCmdlets`, alias and function definitions, `VisibleAliases`, `VisibleFunctions` and the remaining keys; finally enforce input validation and process copy-item function definitions. (Lines 1979–2451; steps summarized from the portions read, remaining details `UNVERIFIED`.) + +## 8. Observable behavior [SPEC] + +- A user connecting to a JEA endpoint sees only the commands listed (`Get-Command` shows them), cannot run others even by full path or alias unless allowed, and runs in the configured language mode (typically `NoLanguage` or `ConstrainedLanguage`), often under a virtual account. +- Inline `FunctionDefinitions` appear as commands; parameter restrictions reject disallowed values at binding time. +- Role membership changes which commands are visible (merge adds commands from every matching role). +- Tests: §16 (JEA tests need an endpoint; most are outside the Pester run). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Errors use ids such as `InvalidRoleDefinitionNotHashtable`, `InvalidRoleValueNotHashtable`, `CouldNotFindRoleCapability`, `CouldNotFindRoleCapabilityFile`, `InvalidRoleCapabilityFileExtension`, `InvalidModulesToImportKeyEntries`, `InvalidVisibleCmdletsKeyEntries`, `InvalidVisibleCommandKeyEntries`; an unknown or wrongly typed key fails validation; a session type value that does not parse fails; a module that is both visible-by-name and not imported fails with a missing-module error; exposing `Import-Module` through visible commands while restricting is guarded (comment at `InitialSessionState.cs` ~L3072). + +## 10. Concurrency and threading + +The configuration is read once per session creation on the endpoint's thread; the resulting `InitialSessionState` is cloned into each runspace; role verification runs per connection. No shared mutable state besides module loading caches. + +## 11. Extension points and customization hooks + +`.pssc` / `.psrc` authoring (`New-PSSessionConfigurationFile`, `New-PSRoleCapabilityFile`), custom `PSSessionConfiguration` classes, `FunctionDefinitions` (wrapper functions that call restricted cmdlets with fixed parameters), `ScriptsToProcess` for startup logic, `RunAsVirtualAccount`/gMSA for identity, `TranscriptDirectory` for auditing, `RequiredGroups` for conditional access. + +## 12. Dependencies + +Inbound: WSMan plugin/SSH subsystem/named-pipe servers (when `-ConfigurationName` or `-ConfigurationFile` is used), `pwsh -ConfigurationFile`, the SDK. Outbound: `InitialSessionState`, module loader (role capability search), command visibility in session state and command discovery, language mode and execution policy, the parser (restricted data evaluation). + +## 13. Performance notes + +Role capability search enumerates module directories per capability name on every connection; configuration parsing is data-only and cheap; large visible-command lists create many entry lookups (`LookUpByName`). + +## 14. Security considerations + +JEA's guarantees rest on a **chain of in-engine checks**: (1) the configuration is parsed as data, not executed; (2) private visibility blocks invocation from the caller's origin (`CommandSearcher.MoveNext` alias visibility, `CommandDiscovery.ShouldRun`, hidden-command errors); (3) restricted language modes prevent script from reaching around the command table (the baseline `RestrictedRemoteServer` is `NoLanguage`); (4) parameter validation constraints narrow what a visible command accepts; (5) transcripts record activity. Weak points visible in the design: visible commands that are themselves powerful (the author's responsibility), proxy functions that forward parameters, visible *external commands*, `Import-Module` exposure, merge semantics that only add, and run-as accounts. The engine does not itself authenticate the caller or decide role membership (the verifier callback and the transport/OS do). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| jea | 246 | +| language-mode | 27 | +| execution-policy | 12 | +| logging | 6 | +| securestring | 1 | +| **total** | 292 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/remoting/commands/CustomShellCommands.cs` | 213 | +| `src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs` | 79 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: restricted sessions are built from a declarative, validated, *deny-by-default* description; visibility is enforced at command resolution and again at invocation; parameter constraints are enforced by the binder, not by wrapper convention; role merging is explicit about allow and deny (deny wins); every restricted session is attributable to an identity and audited. + +## 15. Enterprise considerations + +Security first: §14. Observability: transcripts per session, module/script block logging, endpoint events. Supportability: files are text and versionable (`GUID`, `SchemaVersion`); `Test-PSSessionConfigurationFile` validates. Testability: a configuration can be turned into an `InitialSessionState` in-process (`CreateFromSessionConfigurationFile`) without a remote endpoint. Resource limits: received-object and data-size limits per endpoint class; user-drive size cap. Missing: deny rules and precedence, per-role resource quotas, policy simulation ("what can role X do?") tooling in the engine, signed configuration files. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Modules/Microsoft.PowerShell.Core/PSSessionConfiguration.Tests.ps1` | 7 | 5 | 26 | +| `test/powershell/engine/Remoting/RoleCapabilityFiles.Tests.ps1` | 1 | 0 | 3 | +| **total (2 files)** | 8 | 5 | 29 | + +No xUnit file matched the path keywords. + +Matching is by path keyword (`APPROX`): `SessionConfiguration`, `PSSessionConfiguration`, `RoleCapab`, `JEA`, `ConfigurationFile`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: configurations are data, parsed without code execution, so an endpoint can be reviewed and generated. +- `FACT-SRC`: roles *add* capabilities by concatenation, which keeps the algorithm simple and makes the sum of roles the effective surface. +- `INFERENCE`: using visibility (private vs public) rather than deleting commands lets internal code keep using helpers a caller cannot reach. +- `FACT-SRC`: the restricted baseline starts empty and builds up, instead of removing from a full session. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-JEA-01** [REPLICATE] (area: NLD-SEC) Deny-by-default restricted sessions built from a declarative data file: empty baseline, named visible commands, parameter constraints, language mode, execution policy, startup scripts and transcripts. Evidence: src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.GetInitialSessionState@L1979-L2451. +- **LSN-JEA-02** [REPLICATE] (area: NLD-SEC) Visibility as a per-command property (public/private) enforced at resolution and invocation, so internal helpers stay callable. Evidence: src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.ProcessVisibleCommand@L2614-L2650. +- **LSN-JEA-03** [REDESIGN] (area: NLD-SEC) Add explicit deny rules and a defined precedence (deny over allow) to role merging, plus a "what can this role do" report. DEV: DEV-JEA-220. Evidence: src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.MergeConfigHashIntoConfigHash@L1887-L1929. +- **LSN-JEA-04** [ADAPT] (area: NLD-SEC) Make parameter constraints part of the command contract (typed policy objects) and enforce them in the binder, instead of attribute-name keys in a hashtable. Evidence: src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.ProcessCommandModification@L2527-L2612. +- **LSN-JEA-05** [ADAPT] (area: NLD-SEC) Sign configuration and role files and verify them at load. Evidence: src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCUtils.LoadConfigFile@L1453-L1474. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-070): how `CommandModifications` are applied at run time; `VisibleExternalCommands` mapping; `FunctionDefinitions` proxy generation; `RunAsVirtualAccount` and `GroupManagedServiceAccount` handling (registration cmdlets and native plugin); `RequiredGroups` SDDL building; `ValidateRoleDefinitions` rules; `LoadConfigFile` restricted evaluation; the rest of `GetInitialSessionState` (lines 2200–2450). +- `UNVERIFIED` (VB-071): `PSPrincipal`/`PSSenderInfo` construction and how server code uses them; `New-PSRoleCapabilityFile` and `Test-PSSessionConfigurationFile`. + +Verification: `InitialSessionStateProvider.cs` lines 870–1015 (keys), 1781–1990, 1990–2200 (grep of key lines), 2475–2680 were read; `InitialSessionState.cs` lines 1378–1400 earlier. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/05-remoting/psrp-protocol.md b/docs-out/05-remoting/psrp-protocol.md new file mode 100644 index 0000000..cfe9256 --- /dev/null +++ b/docs-out/05-remoting/psrp-protocol.md @@ -0,0 +1,200 @@ +--- +id: REM-PSRP-001 +title: PowerShell Remoting Protocol (PSRP) layers +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: OUT-OF-SCOPE-DOCUMENTED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/remoting/common/WireDataFormat/EncodeAndDecode.cs, src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemotingDataObject.cs, src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteSessionCapability.cs, src/System.Management.Automation/engine/remoting/common/fragmentor.cs, src/System.Management.Automation/engine/remoting/common/misc.cs, src/System.Management.Automation/engine/remoting/server/serverremotesession.cs, src/System.Management.Automation/engine/remoting/server/ServerRemotingProtocol2.cs, src/System.Management.Automation/engine/remoting/server/serverremotingprotocolimplementation.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/remoting/_folder.md] +generated_from: [data/types.json, data/enums.json, data/tests.json, data/enforcement-points.json, data/conditional-compilation.json] +--- + +# PowerShell Remoting Protocol (PSRP) layers + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `OUT-OF-SCOPE-DOCUMENTED`: remoting is documented so its design lessons and protocol shape are known, not to be replicated in the first new-shell version. Transports are in [transports](transports.md); the object serialization used inside messages is in [serialization-clixml](serialization-clixml.md); constrained endpoints in [jea](jea.md). + +## 1. Purpose and scope [SPEC] + +PSRP lets a client engine drive a runspace pool and pipelines hosted by a remote (or child-process) engine. The protocol has four layers: **transport** (moves bytes), **fragmentation** (splits messages into blocks), **message framing** (a typed header plus a serialized payload), and **session/runspace/pipeline semantics** (state machines and message types). This document describes the last three; the first is in the transports document. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/remoting/common/WireDataFormat/EncodeAndDecode.cs | enums, constants, encoders/decoders | message type ids, data names, version constants, payload builders | B | +| src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemotingDataObject.cs@L15-L268 | class | one protocol message (header + payload) | B | +| src/System.Management.Automation/engine/remoting/common/fragmentor.cs::FragmentedRemoteObject@L29-L426 | class | one fragment (header + blob) | B | +| src/System.Management.Automation/engine/remoting/common/fragmentor.cs::SerializedDataStream@L433-L957 | class | writes a message as fragments | B | +| src/System.Management.Automation/engine/remoting/common/fragmentor.cs::Fragmentor@L965-L1125 | class | fragment size, serialization context, (de)serialization entry points | B | +| src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteSessionCapability.cs::RemoteSessionCapability@L19-L91 | class | protocol, PS and serialization versions | B | +| src/System.Management.Automation/engine/remoting/client/clientremotesession.cs, clientremotesessionprotocolstatemachine.cs | classes | client session and state machine | B | +| src/System.Management.Automation/engine/remoting/server/serverremotesession.cs::ServerRemoteSession@L70-L1094 | class | server session, negotiation | B | +| src/System.Management.Automation/engine/remoting/server/ServerRunspacePoolDriver.cs, ServerPowerShellDriver.cs | classes | server-side runspace pool and pipeline drivers | B | +| src/System.Management.Automation/engine/remoting/client/RemoteRunspacePoolInternal.cs, remotepipeline.cs, ClientRemotePowerShell.cs | classes | client-side counterparts | B | +| src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHost*.cs, host/RemoteHostMethodInfo.cs | classes | host method calls over the wire | B | + +## 3. Responsibilities and non-responsibilities + +Owns: message ids and framing, version negotiation, session/runspace/pipeline message flow, fragmentation, host-call forwarding. Does not own: byte transport (WSMan, SSH, pipes), CLIXML details, authentication (transport), endpoint configuration (JEA). + +## 4. Key types and contracts [IMPL] + +- **Protocol versions** (`RemotingConstants`): `2.0` (Windows 7 RC), `2.1` (Windows 7 RTM), `2.2` (Windows 8 RTM), `2.3` (Windows 10 RTM; enables the information stream), `2.4` (PowerShell 7.6); `ProtocolVersionCurrent = 2.4`. Version 2.4 **deprecates the session-key exchange**: when both sides are 2.4 or later the messages `PUBLIC_KEY`, `PUBLIC_KEY_REQUEST` and `ENCRYPTED_SESSION_KEY` are obsolete and `SecureString` values are no longer encrypted at the protocol layer; the source comment states the RSA padding used before is not secure and that PSRP must therefore run on a secure transport (src/System.Management.Automation/engine/remoting/common/WireDataFormat/EncodeAndDecode.cs::RemotingConstants@L75-L113). +- **Negotiation rule** (`RunServerNegotiationAlgorithm`): the server accepts when the client protocol version has the same *major* and a *minor* at least the server's (so a 2.4 server accepts clients ≥ 2.4), or — in the normal (non-connect) case — when the client is exactly 2.0, 2.1, 2.2 or 2.3 (the server then lowers itself to the client's version); in the *connect* (reconnect) case only 2.2 and 2.3 are accepted this way. It also requires the client's PS version to have the same major and a minor ≥ the server's, and the client's serialization version the same major and a minor ≥ the server's; any violation is a `PSRemotingDataStructureException` (`ServerNegotiationFailed`) naming the offending item (src/System.Management.Automation/engine/remoting/server/serverremotesession.cs::ServerRemoteSession.RunServerNegotiationAlgorithm@L967-L1058). The client's PS version field is a fixed `2.0` in the capability object (comment: "PSVersionInfo.PSVersion"). +- **Message** = header + payload: `Destination` (uint, little-endian: `Client=1`, `Server=2`, `Listener=4`), `DataType` (uint, little-endian, the id below), `RunspacePoolId` (16-byte GUID), `PowerShellId` (16-byte GUID; empty GUID for non-pipeline messages), then the payload as **CLIXML** (UTF-8 XML fragment without declaration, depth 1) when present (src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemotingDataObject.cs@L178-L214). `RemotingTargetInterface`: `Session=1`, `RunspacePool=2`, `PowerShell=3`. +- **Message type ids** (`RemotingDataType`; the high bits group them by target): *session* `0x0001xxxx` — `SessionCapability 0x00010002`, `CloseSession 0x00010003`, `CreateRunspacePool 0x00010004`, `PublicKey 0x00010005`, `EncryptedSessionKey 0x00010006`, `PublicKeyRequest 0x00010007`, `ConnectRunspacePool 0x00010008`; *runspace pool* `0x0002xxxx` — `SetMaxRunspaces 0x00021002`, `SetMinRunspaces 0x00021003`, `RunspacePoolOperationResponse 0x00021004`, `RunspacePoolStateInfo 0x00021005`, `CreatePowerShell 0x00021006`, `AvailableRunspaces 0x00021007`, `PSEventArgs 0x00021008`, `ApplicationPrivateData 0x00021009`, `GetCommandMetadata 0x0002100A`, `RunspacePoolInitData 0x0002100B`, `ResetRunspaceState 0x0002100C`, `RemoteHostCallUsingRunspaceHost 0x00021100`, `RemoteRunspaceHostResponseData 0x00021101`; *pipeline* `0x0004xxxx` — `PowerShellInput 0x00041002`, `PowerShellInputEnd 0x00041003`, `PowerShellOutput 0x00041004`, `PowerShellErrorRecord 0x00041005`, `PowerShellStateInfo 0x00041006`, `PowerShellDebug 0x00041007`, `PowerShellVerbose 0x00041008`, `PowerShellWarning 0x00041009`, `PowerShellProgress 0x00041010`, `PowerShellInformationStream 0x00041011`, `StopPowerShell 0x00041012`, `RemoteHostCallUsingPowerShellHost 0x00041100`, `RemotePowerShellHostResponseData 0x00041101`. +- **Fragmentation**: the header of each fragment is 21 bytes, big-endian: object id (8 bytes, sign bit 0), fragment id (8 bytes, sign bit 0), a flags byte (`S=0x1` start fragment, `E=0x2` end fragment), blob length (4 bytes), then the blob. A message gets one object id; fragment ids count from 0 (`IsStartFragment` is `fragmentId == 0`); the last fragment carries the end flag. The default **fragment size is 32 KB** (`BaseTransportManager.DefaultFragmentSize = 32 << 10`); the blob size per fragment is the fragment size minus the header. Serialization depth for remoting is **1** (`SerializationDepthForRemoting`) with the remoting serialization options (src/System.Management.Automation/engine/remoting/common/fragmentor.cs::FragmentedRemoteObject@L29-L426; src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs::BaseTransportManager.DefaultFragmentSize@L221-L221). +- **Session state machine** (`RemoteSessionState`): `UndefinedState`, `Idle`, `Connecting`, `Connected`, `NegotiationSending`, `NegotiationSendingOnConnect`, `NegotiationSent`, `NegotiationReceived`, `NegotiationPending`, `ClosingConnection`, `Closed`; events (`RemoteSessionEvent`, 32 values) include `CreateSession`, `ConnectSession`, `NegotiationSending`, `NegotiationSendCompleted`, `NegotiationReceived`, `NegotiationCompleted`, `Close`, `CloseCompleted`, `CloseFailed`, `ConnectFailed`, `NegotiationFailed`, `NegotiationTimeout`, `SendFailed`, `ReceiveFailed`, `FatalError`, `MessageReceived`, key-exchange events (`KeySent`, `KeyReceived`, `KeyRequested`, …) and disconnect/reconnect events (`DisconnectStart`, `DisconnectCompleted`, `ReconnectStart`, …) (src/System.Management.Automation/engine/remoting/common/remotesession.cs, misc.cs; the transition table was not read, `UNVERIFIED`). +- **Server dispatch**: session-level messages are handled by the server session (`CreateRunspacePool`, `CloseSession`, `SessionCapability`, `PublicKey`); runspace-pool and pipeline messages go to the runspace pool driver: `CreatePowerShell`, `GetCommandMetadata`, `RemoteRunspaceHostResponseData`, `SetMaxRunspaces`, `SetMinRunspaces`, `AvailableRunspaces`, `ResetRunspaceState` (pool level) and `StopPowerShell`, `PowerShellInput`, `PowerShellInputEnd`, `RemotePowerShellHostResponseData` (pipeline level) (src/System.Management.Automation/engine/remoting/server/ServerRemotingProtocol2.cs@L121-L660). + +## 5. Data model + +Payload property names are strings in `RemoteDataNameStrings`: for example `MinRunspaces`, `MaxRunspaces`, `PSThreadOptions`, `ApartmentState`, `HostInfo`-style host data, `PowerShell` with `IsNested`, `Cmds` (command list: `Cmd` text, `IsScript`, `UseLocalScope`, `Args` with `N` name and `V` value, and the merge flags `MergeMyResult`, `MergeToResult`, `MergeError`, `MergeWarning`, `MergeVerbose`, `MergeDebug`, `MergeInformation`, `MergePreviousResults`), `NoInput`, `PSInvocationSettings`, `RemoteStreamOptions`, `AddToHistory`, `RedirectShellErrorOutputPipe`, `PipelineState`, `RunspaceState`, `ExceptionAsErrorRecord`, the event-argument fields `PSEventArgs.*`, host-call fields `ci` (call id), `mi` (method id), `mp` (parameters), `mr` (return value), `me` (exception), and capability fields `protocolversion`, `PSVersion`, `SerializationVersion`, `TimeZone`. Notes added to remote objects: `PSComputerName`, `RunspaceId`, `PSShowComputerName`, `PSSourceJobInstanceId`, `PSEventObject` (`RemotingConstants`). + +## 6. Control flow + +```mermaid +sequenceDiagram + participant C as Client engine + participant T as Transport + participant S as Server session + participant D as Runspace pool driver + C->>T: SESSION_CAPABILITY (protocol, PS and serialization versions) + T->>S: fragments, reassembled + S->>C: SESSION_CAPABILITY (negotiated) + C->>S: CREATE_RUNSPACEPOOL (min, max, thread options, host info, application arguments) + S->>D: create runspace pool + S->>C: APPLICATION_PRIVATE_DATA, RUNSPACEPOOL_STATE (Opened) + C->>D: CREATE_PIPELINE (commands, settings) + C->>D: PIPELINE_INPUT, PIPELINE_INPUT_END + D->>C: PIPELINE_OUTPUT, ERROR_RECORD, DEBUG, VERBOSE, WARNING, PROGRESS, INFORMATION + D->>C: PIPELINE_HOST_CALL (prompt) + C->>D: PIPELINE_HOST_RESPONSE + D->>C: PIPELINE_STATE (Completed, Failed or Stopped) + C->>S: CLOSE_SESSION +``` + +Ordered: (1) transport connects; (2) both sides exchange `SessionCapability` and run the negotiation rule; (3) the client sends `CreateRunspacePool`; the server creates the pool from the configured initial session state and answers with private data and state; (4) each `CreatePowerShell` creates a server pipeline driver bound to the pipeline GUID; input is streamed until `PowerShellInputEnd`; output and stream records flow back as typed messages; host calls (prompts, `Write-Host`, `ReadLine`) travel as `RemoteHostCall…` and wait for `…HostResponseData`; (5) pipeline state ends the pipeline; (6) `CloseSession` ends the session. The pre-2.4 key exchange (`PublicKey`, `EncryptedSessionKey`, `PublicKeyRequest`) occurs only when either side is older than 2.4 (`UNVERIFIED` exact conditions). + +## 7. Algorithms + +Version negotiation: §4. Fragmentation: serialize the message into a stream that cuts a fragment whenever the current blob reaches `fragmentSize - 21` bytes, assigning increasing fragment ids; the receiver reassembles by object id until the end flag. Priority: outgoing data is kept in one serialized-data store per priority (`DataPriorityType`: `Default = 0`, `PromptResponse = 1`; `PrioritySendDataCollection`) so host prompt answers are not stuck behind bulk output (the scheduling between the stores was not read, `UNVERIFIED`). + +## 8. Observable behavior [SPEC] + +- A remote command's output appears as deserialized objects with extra properties (`PSComputerName`, `RunspaceId`); streams (error, warning, verbose, debug, progress, information) are relayed; prompts from the remote script are shown on the client host. +- Version mismatch yields a clear negotiation error naming the field and versions. +- `Stop` on the client sends `StopPowerShell`; the server stops the pipeline. +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`PSRemotingDataStructureException` for malformed or unexpected messages (wrong destination or type), `PSRemotingTransportException` for transport faults, `PSRemotingDataStructureException(ServerNegotiationFailed / ServerConnectFailedOnNegotiation)` for version failures; a message whose destination is not `Server` (on a server) or whose type is unexpected for the session state is rejected; fragments out of order (`FragmentId` mismatch) fail reassembly (`UNVERIFIED` code path); deserialization of a null payload is an error (`DeserializedObjectIsNull`). + +## 10. Concurrency and threading + +One session has one transport manager per runspace pool and one per command (pipeline); sends are serialized per transport; receive callbacks run on transport threads and hand off to the session state machine and drivers; server pipelines run on runspace threads (see [runspace-and-pipeline](../03-runtime/runspace-and-pipeline.md)); the throttle manager limits concurrent remote operations on the client (`throttlemanager.cs`). + +## 11. Extension points and customization hooks + +Protocol version constants (new minor per change); `RemoteHostMethodId`-based host calls; application private data and application arguments (`$PSSenderInfo`, `PSApplicationPrivateData`) exchanged at session creation; session configuration (endpoints) selecting the initial session state; custom transports implement the transport-manager contract. + +## 12. Dependencies + +Inbound: remoting cmdlets (`Invoke-Command`, `New-PSSession`, `Enter-PSSession`), jobs, `PowerShell`/`Runspace` APIs with remote connection info. Outbound: CLIXML serializer, transport managers, runspace pool/pipeline (server side), host (host calls), crypto helper (pre-2.4 key exchange). + +## 13. Performance notes + +32 KB fragments and Base64-wrapped transport frames (out-of-process transports) add overhead; serialization depth 1 limits object graph size; large outputs are streamed per object; pool and pipeline messages are small relative to CLIXML payloads. + +## 14. Security considerations + +PSRP itself provides **no confidentiality or authentication from version 2.4**: it relies on the transport (WSMan with Kerberos/HTTPS, SSH, local pipes). The deprecated session-key exchange used a padding the source calls insecure. Enforcement points here: negotiation rejects incompatible versions; the server session validates message destinations and types per state; deserialization of payloads uses restricted CLIXML types (deserialized objects are data-only, see CLIXML document); the server applies the endpoint's initial session state and language mode to the runspace pool; `$PSSenderInfo` identifies the caller to server-side code. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| jea | 59 | +| securestring | 21 | +| language-mode | 18 | +| execution-policy | 12 | +| logging | 6 | +| app-control | 5 | +| remoting-deser | 2 | +| **total** | 123 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs` | 79 | +| `src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHost.cs` | 7 | +| `src/System.Management.Automation/engine/remoting/fanin/PSSessionConfigurationData.cs` | 5 | +| `src/System.Management.Automation/engine/remoting/common/PSETWTracer.cs` | 4 | +| `src/System.Management.Automation/engine/remoting/fanin/WSManTransportManager.cs` | 4 | +| `src/System.Management.Automation/engine/remoting/server/ServerRunspacePoolDriver.cs` | 4 | +| `src/System.Management.Automation/engine/remoting/server/serverremotesession.cs` | 4 | +| `src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHostEncoder.cs` | 3 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here (if remoting is ever built): the transport authenticates both peers and encrypts; the protocol never sends secrets in a weaker envelope than the transport; the server side applies policy before creating the runspace pool and logs every session with identity; message size and rate limits protect the server. + +## 15. Enterprise considerations + +Security first: §14. Observability: ETW tracers for remoting (`PSETWTracer`), session/runspace/pipeline ids as GUIDs, state events. Supportability: explicit protocol versions with a documented change history; `PSSessionOption` timeouts. Testability: protocol layers are separable (fragmentor, encoders) but tests are mostly end-to-end (§16). Resource limits: runspace pool min/max, fragment size, timeouts and size limits are transport constants and options: maximum received object size 10 MB (`MaximumReceivedObjectSize = 10 << 20`), connection open timeout 3 minutes, operation timeout 3 minutes, close timeout 1 minute, server keep-alive 4 minutes, idle timeout default "use the server's" (src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs@L209-L225, src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs@L206-L268). Missing: protocol-level authentication, message signing, per-message size caps independent of transport. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/SSHRemoting/SSHRemoting.Basic.Tests.ps1` | 1 | 2 | 19 | +| `test/powershell/engine/Remoting/CustomConnection.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/Remoting/InvokeCommandRemoteDebug.Tests.ps1` | 1 | 0 | 4 | +| `test/powershell/engine/Remoting/PSSession.Tests.ps1` | 2 | 0 | 2 | +| `test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1` | 5 | 0 | 19 | +| `test/powershell/engine/Remoting/RemoteSession.Disconnect.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/engine/Remoting/RoleCapabilityFiles.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/Remoting/RunspacePool.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/engine/Remoting/SSHRemotingAPI.Tests.ps1` | 1 | 1 | 3 | +| `test/powershell/engine/Remoting/SSHRemotingCmdlets.Tests.ps1` | 4 | 0 | 4 | +| `test/powershell/engine/Remoting/SessionOption.Tests.ps1` | 1 | 0 | 4 | +| **total (11 files)** | 19 | 3 | 63 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_RemoteHyperV.cs` | 0 | 0 | `RemoteHyperVTests`, `of` | + +Matching is by path keyword (`APPROX`): `Remoting/`, `engine/Remoting`, `test_RemoteHyperV`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: a typed binary header around XML payloads keeps routing cheap and payloads self-describing. +- `FACT-SRC`: protocol minor versions are bumped for each change and old versions stay supported by down-negotiation. +- `FACT-SRC`: encryption was moved out of the protocol (2.4) because the in-protocol scheme was weak; transport security became mandatory in practice. +- `INFERENCE`: two priority streams exist so prompt responses are not blocked by large output. +- `FACT-SRC`: host methods are forwarded as RPCs so remote scripts can use the local UI. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-PSRP-01** [AVOID] (area: NLD-SEC) Do not build remoting in the first version; if built later, do not copy PSRP's XML payload design or its in-protocol key exchange. DEV: DEV-REM-210. Evidence: src/System.Management.Automation/engine/remoting/common/WireDataFormat/EncodeAndDecode.cs::RemotingConstants@L75-L113. +- **LSN-PSRP-02** [ADAPT] (area: NLD-SEC) Keep the lesson that protocol versions are explicit and negotiated downward with a published history, and that security belongs to a mutually authenticated encrypted transport. Evidence: src/System.Management.Automation/engine/remoting/server/serverremotesession.cs::ServerRemoteSession.RunServerNegotiationAlgorithm@L967-L1058. +- **LSN-PSRP-03** [ADAPT] (area: NLD-RUN) Keep length-prefixed fragmentation with start/end flags and a priority lane for control and prompt traffic. Evidence: src/System.Management.Automation/engine/remoting/common/fragmentor.cs::FragmentedRemoteObject@L29-L426. +- **LSN-PSRP-04** [ADAPT] (area: NLD-RUN) Forward host calls (prompts, progress) as typed RPCs with call ids if a remote host UI is ever needed. Evidence: src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHost.cs@L1-L762. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-064): the session state-machine transition tables (client and server), key-exchange conditions, runspace-pool and pipeline driver message handling (`ServerRunspacePoolDriver`, `ServerPowerShellDriver`), client-side classes, `RemoteHost` encoding, `EncodeAndDecode` payload builders, `ThrottleManager`. +- `UNVERIFIED` (VB-065): reassembly error paths in `Defragmentor`, data size limits and timeouts, `PSSenderInfo` construction. + +Verification: `EncodeAndDecode.cs` (`RemotingConstants`, type id lists, selected enums), `RemotingDataObject.cs` lines 150–309, `fragmentor.cs` lines 29–280 and 965–1125, `serverremotesession.cs` lines 300–360 and 967–1058, `ServerRemotingProtocol2.cs` case labels, `misc.cs` enums, `BaseTransportManager.cs` line 221 were read; the rest is from signatures. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/05-remoting/serialization-clixml.md b/docs-out/05-remoting/serialization-clixml.md new file mode 100644 index 0000000..46c2dc1 --- /dev/null +++ b/docs-out/05-remoting/serialization-clixml.md @@ -0,0 +1,175 @@ +--- +id: REM-CLIXML-001 +title: Serialization (CLIXML) and type fidelity +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/serialization.cs, src/System.Management.Automation/engine/SerializationStrings.cs, src/System.Management.Automation/engine/remoting/common/fragmentor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/types.json, data/enums.json, data/tests.json, data/enforcement-points.json] +--- + +# Serialization (CLIXML) and type fidelity + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED` (used by remoting, jobs, `Export-Clixml`/`Import-Clixml`, `Start-Job`). Object model behind it: [extended-type-system](../03-runtime/extended-type-system.md). Wire use: [psrp-protocol](psrp-protocol.md). + +## 1. Purpose and scope [SPEC] + +CLIXML is PowerShell's XML object format. It turns an object graph into text and back, preserving *enough* information that a script on the other side can use the result: values of well-known types round-trip exactly, other objects become property bags that carry their type names and a string form. This document covers the element vocabulary, the serializer and deserializer rules (depth, references, string forms), the **type-fidelity rules** (what survives, what is lost), the special handling of secure strings and script blocks, and the safety settings of the XML readers. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/serialization.cs::PSSerializer@L86-L216 | public static class | `Serialize(object[, depth])`, `Deserialize(string)`, `DeserializeAsList` | B | +| src/System.Management.Automation/engine/serialization.cs::Serializer@L221-L332, Deserializer@L493-L823 | classes | public-ish drivers over an `XmlWriter`/`XmlReader` | B | +| src/System.Management.Automation/engine/serialization.cs::SerializationContext@L46-L81, DeserializationContext@L345-L399 | classes | depth, option flags, crypto helper | B | +| src/System.Management.Automation/engine/serialization.cs::InternalSerializer@L837-L2961 | class | the writer: `WriteOneObject`, containers, PSObject handling | B | +| src/System.Management.Automation/engine/serialization.cs::InternalDeserializer@L2966-L4996 | class | the reader: `ReadOneObject`, known types, containers | B | +| src/System.Management.Automation/engine/serialization.cs::KnownTypes@L5167-L5376 | static class | table of primitive types, tags and (de)serializers | B | +| src/System.Management.Automation/engine/serialization.cs::DeserializingTypeConverter@L6670-L7678 | public sealed class | rehydrates selected types from deserialized property bags | B | +| src/System.Management.Automation/engine/serialization.cs::PSPrimitiveDictionary@L5952-L6644 | public class | a hashtable restricted to primitive values (used for application arguments/private data) | B | +| src/System.Management.Automation/engine/SerializationStrings.cs::SerializationStrings@L9-L311 | constants | element and attribute names | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the CLIXML grammar, depth and reference rules, the known-type table, rehydration, and reader safety settings. Does not own: the transport framing of serialized text (PSRP fragments), encryption keys (`PSRemotingCryptoHelper`, remoting), the type-data settings themselves (`SerializationDepth`, `SerializationMethod` in type data, see the ETS document). + +## 4. Key types and contracts [IMPL] + +- **Vocabulary** (`SerializationStrings`): root `Objs` (attributes `Version`), object `Obj` (attributes `RefId`, optional name `N`, stream `S`), type names `TN` (child `T` items) and back-reference `TNRef`, property sections `Props` (adapted properties) and `MS` (member set: extended note properties), note property `N`, string form `ToString`, containers `LST` (list), `IE` (enumerable), `DCT` with `En`/`Key`/`Value` (dictionary), `STK` (stack), `QUE` (queue), object back-reference `Ref`, nil `Nil`; primitive tags: `S` string, `C` char, `B` bool, `By` byte, `SB` sbyte, `I16`/`U16`, `I32`/`U32`, `I64`/`U64`, `Sg` single, `Db` double, `D` decimal, `DT` datetime, `TS` timespan, `G` guid, `BA` byte array (base64), `URI`, `Version`, `SemanticVersion`, `XD` XML document, `SBK` script block, `SS` secure string, `PR` progress record (with child tags `AI`, `PI`, `AV`, `SD`, `CO`, `PC`, `SR`, `T`); namespace `http://schemas.microsoft.com/powershell/2004/04` (prefix `ps`) (src/System.Management.Automation/engine/SerializationStrings.cs). +- **Options** (`SerializationOptions`): `UseDepthFromTypes`, `NoRootElement`, `NoNamespace`, `NoObjectRefIds`, `PreserveSerializationSettingOfOriginal`; `RemotingOptions` = `UseDepthFromTypes | NoRootElement | NoNamespace | PreserveSerializationSettingOfOriginal`. `DeserializationOptions`: `NoRootElement`, `NoNamespace`, `DeserializeScriptBlocks`, `RemotingOptions` = `NoRootElement | NoNamespace`. **Depth**: `SerializationContext` requires depth ≥ 1; the default context depth is 2 (`DefaultSerializationDepth`), `PSSerializer.Serialize(obj)` uses depth **1** (`s_mshDefaultSerializationDepth`), remoting uses depth **1** (`SerializationDepthForRemoting`); nesting is hard-capped at **50** levels below the top regardless of depth (`MaxDepthBelowTopLevel`), where the writer emits the too-deep message string instead of the object. +- **`WriteOneObject` algorithm** (src/System.Management.Automation/engine/serialization.cs::InternalSerializer.WriteOneObject@L971-L1042): (1) `null` → `Nil`; (2) increment the nesting counter; if at the 50-level cap write the "too deep" string and stop; (3) compute the depth for this object (`GetDepthOfSerialization`: `CimInstance`, `PSCredential` and `SwitchParameter` use 1, `PSSenderInfo` uses 4, a depth from type data when `UseDepthFromTypes` and the type data gives a positive value, and a deserialized object whose remaining depth is ≤ 0 uses 1 when `PreserveSerializationSettingOfOriginal`); (4) a primitive known type wrapped in a `PSObject` with extra members is written as an `Obj` with the primitive inside; (5) if the object was already written, emit `` (reference ids make shared and cyclic graphs finite; disabled by `NoObjectRefIds`); (6) a plain primitive known type is written as its tag element; (7) known containers (dictionary → `DCT`, other enumerable → `LST`/`IE`, stack, queue) are written with their elements at depth-1; (8) if the depth is 0 or the type data says **serialize as string** (`SerializationMethod.String`), write the object as its string form (the `ToString` representation) inside `Obj`; (9) otherwise write a complex object through `HandleComplexTypePSObject` (not read, `UNVERIFIED`; `INFERENCE` from the reader's expectations: `TN` type names, `Props` adapted properties at depth-1, `MS` extended properties and a `ToString` string form). +- **Known primitive types** are exact round-trip types (`KnownTypes` table keyed by full type name and by tag): booleans, integer widths, floating types, decimal, char, string, byte array, date/time, time span, GUID, URI, version, semantic version, script block (as text), XML document, secure string, progress record. Enums are written as their underlying value plus type names (`UNVERIFIED` exact handling). +- **Deserializer** (`InternalDeserializer`): `ReadOneObject` reads `Obj` into a `PSObject` (`ReadPSObject`): type names (the first name gets the **`Deserialized.` prefix**, e.g. `Deserialized.System.IO.FileInfo`; if absent the original names stay), `Props` become note properties (read-only data), `MS` populates instance members, a `ToString` element sets the string form and adds a `ToString` method member, and the inner element is read as a primitive, a container (`ReadListContainer`/`ReadDictionary`; the resulting collection types were not read, `UNVERIFIED`) or a nested object; unknown tags raise an XML exception unless unknown tags are allowed (then skipped). The resulting object is *data only*: it is flagged as deserialized and has no methods of the original type. +- **Rehydration** (`DeserializingTypeConverter`, a `PSTypeConverter`): when a script or the engine converts a deserialized object to a specific type, a table maps selected target types to reconstruction functions: `PSPrimitiveDictionary`, `SwitchParameter`, `PSListModifier`, `PSCredential`, `PSSenderInfo`, `CultureInfo`, `ParameterSetMetadata`, `X509Certificate2`, `X500DistinguishedName`, `IPAddress`, `MailAddress`, `DirectorySecurity`/`FileSecurity`/`RegistrySecurity`, the format-data object model (`ExtendedTypeDefinition`, `FormatViewDefinition`, `PSControl`, table/list/wide/custom control classes, `DisplayEntry`, `EntrySelectedBy`) and `CompletionResult`; other types stay property bags (src/System.Management.Automation/engine/serialization.cs::DeserializingTypeConverter@L6670-L7678). +- **Secure strings**: written as `SS` with the text *encrypted* — with the remoting crypto helper if one is in the context, else with the platform data-protection helper (`SecureStringHelper`; Windows DPAPI-style, on Unix a different scheme, `UNVERIFIED`); on read the same helper decrypts (failure → "deserialize secure string failed"). Since protocol 2.4 remoting skips the session-key encryption and relies on the transport (see PSRP document). +- **Script blocks**: serialized as text; deserialized back to a `ScriptBlock` only when `DeserializeScriptBlocks` is set, otherwise as a string — so script blocks do not become executable from untrusted data by default (`UNVERIFIED` default path). +- **Reader safety**: the CLIXML reader prohibits DTDs, has no XML resolver, ignores comments and processing instructions, does not check characters, limits entity expansion to 1,024 characters; the reader for *untrusted XML documents* (`XD` values) allows DTD parsing but with an entity limit of 1,024 characters and a document limit of 512M characters (src/System.Management.Automation/engine/serialization.cs::InternalDeserializer.GetXmlReaderSettingsForCliXml@L4172-L4191). + +## 5. Data model + +Serializer state: the `XmlWriter`, the context (depth, options, crypto helper, CIM class cache), a reference-id table (`ReferenceIdHandlerForSerializer`: object → id), the current nesting counter, an optional `TypeTable` (type data lookup) and a stop flag. Deserializer state: the `XmlReader`, a reference table (id → object), a type-name table (for `TNRef`), the context and the same `TypeTable`. + +## 6. Control flow + +```mermaid +flowchart TD + O[object] --> N{null?} + N -- yes --> NIL[Nil] + N -- no --> M{nesting at 50?} + M -- yes --> TD[too-deep string] + M -- no --> D[effective depth] + D --> R{already written?} + R -- yes --> REF[Ref RefId] + R -- no --> P{primitive known type?} + P -- yes --> PT[tag element] + P -- no --> C{container?} + C -- yes --> CT[LST, IE, DCT, STK, QUE with elements at depth-1] + C -- no --> S{depth 0 or serialize-as-string?} + S -- yes --> STR[Obj with string form] + S -- no --> X[Obj: TN, Props, MS, ToString] +``` + +## 7. Algorithms + +`WriteOneObject` (§4) is the algorithm; the reader mirrors it. Type-name prefix: `Deserialized.` is added to the first type name when reading an object whose type is not a known primitive; helper methods in the same file add and strip the prefix (lines 682–820) and the format system uses the stripped names as a fallback for view selection (see [formatting-and-output](../03-runtime/formatting-and-output.md) §4). + +## 8. Observable behavior [SPEC] + +- A process object sent through remoting arrives as `Deserialized.System.Diagnostics.Process` with its properties (depth 1: nested objects become strings at depth 0) and prints using its `ToString` or default display; method calls on it fail. +- `Export-Clixml`/`Import-Clixml` and `[System.Management.Automation.PSSerializer]::Serialize` produce and consume the same format; `-Depth` controls nesting (deeper values become strings). +- Primitive values, GUIDs, dates, versions, URIs and byte arrays round-trip as the same .NET types (they are in the known-type table); `EXTERNAL` knowledge (not verified in source here): hashtables come back as hashtables and arrays as object arrays; custom objects come back as property bags. +- Secure strings round-trip only on the same machine/user (file case) or under the same session key/transport (remoting). +- Tests: §16. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Malformed XML or unexpected elements → `XmlException` with line information (`NewXmlException`); version attribute checked (`ValidateVersion`); too deep → the "deserialization too deep" string; cyclic graphs terminate through references; failed secure-string decryption → error; stop requests are honoured between objects (`CheckIfStopping`); unknown CIM class ids are looked up in per-stream caches (`CimClassSerializationCache`). + +## 10. Concurrency and threading + +A serializer or deserializer instance is used by one thread; the stop flag is set from another thread (`Stop()`); `KnownTypes` and the `TypeTable` are shared read-only; `WeakReferenceDictionary` helps reference tables avoid leaks. + +## 11. Extension points and customization hooks + +Type data: `SerializationDepth`, `SerializationMethod` (String or default), `PropertySerializationSet` (only listed properties are serialized), `TargetTypeForDeserialization`; `DeserializingTypeConverter` table (internal); `-Depth` on `Export-Clixml`/`ConvertTo-Json`-style cmdlets; `PSSerializer.DeserializeAsList`; `PSPrimitiveDictionary` for safe application data. + +## 12. Dependencies + +Inbound: remoting (every message payload), `Export-Clixml`/`Import-Clixml`, background jobs (parameters and results), `PSSerializer`, debugger/event forwarding. Outbound: ETS (`PSObject`, type table, adapters), `LanguagePrimitives`, XML APIs, the crypto helper, CIM types (`CimInstance` special cases). + +## 13. Performance notes + +Text XML is verbose; depth 1 limits size; reference ids avoid duplicating shared sub-objects; `KnownTypes` lookups are dictionary-based; secure-string encryption per value is costly; very large byte arrays are Base64 (+33 %). Writers avoid indentation (`Indent = false`) and character checking is off. + +## 14. Security considerations + +Deserialization is a classic attack surface: this implementation limits it by design — payloads become inert property bags; rehydration is limited to a fixed allow-list of types (`DeserializingTypeConverter` table) and never instantiates arbitrary types from type names; script blocks are not revived unless asked; DTDs are prohibited in CLIXML and entity expansion is capped; nesting is capped at 50. Risks that remain: allocation attacks through large documents (the CLIXML reader sets no document-size cap in the settings read; the untrusted-XML reader caps at 512M characters), rehydration functions that parse text (certificates, security descriptors, IP addresses) and secure-string handling. Serialization can also leak data: depth-1 property bags include whatever properties the object's adapter exposes (paths, tokens in properties). + +### Security surface + + +| Category | Seed hits | +|---|---:| +| remoting-deser | 75 | +| securestring | 23 | +| **total** | 98 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/serialization.cs` | 97 | +| `src/System.Management.Automation/engine/SerializationStrings.cs` | 1 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: deserialization never instantiates types by name; the set of rehydratable types is a fixed allow-list; size, depth and entity limits apply to every reader; secrets are marked sensitive and never serialized in clear text; a deserialized object cannot be passed where a live object with methods is required without an explicit, policy-checked conversion. + +## 15. Enterprise considerations + +Security first: §14. Observability: ETW analytic events for depth overrides, mode overrides and max-depth hits. Supportability: a documented schema and version attribute; text format is inspectable. Testability: round-trip tests (§16) and `PSSerializer` as a public API. Resource limits: depth and nesting caps; no byte cap on CLIXML input in the settings read. Missing: a binary or schema-validated format, streaming parse limits, content signing. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/engine/Api/Serialization.Tests.ps1` | 1 | 0 | 7 | +| **total (1 files)** | 1 | 0 | 7 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_Serialization.cs` | 2 | 0 | `SerializationTests` | + +Matching is by path keyword (`APPROX`): `Serialization`, `Export-Clixml`, `Import-Clixml`, `CliXml`, `Clixml`, `test_Serialization`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: fidelity is *selective* — exact for primitives, approximate (property bag + string) for everything else, because live objects cannot cross process boundaries. +- `FACT-SRC`: the default depth is small (1 for remoting and `PSSerializer`, 2 for the internal default) to bound size; type data can raise it for important types. +- `INFERENCE`: keeping `ToString` in the stream preserves readable output on the receiving side. +- `FACT-SRC`: the `Deserialized.` prefix lets format data and type data treat deserialized objects separately from live ones. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-CLIXML-01** [ADAPT] (area: NLD-TYP) Keep the idea of a self-describing, depth-limited, reference-preserving interchange format with exact primitives and inert property bags for the rest, but choose a schema'd format (for example JSON or CBOR with a type vocabulary) and keep CLIXML import/export as a compatibility adapter. Evidence: src/System.Management.Automation/engine/serialization.cs::InternalSerializer.WriteOneObject@L971-L1042. +- **LSN-CLIXML-02** [REPLICATE] (area: NLD-SEC) Never instantiate types from payload type names; rehydrate through a fixed allow-list only. Evidence: src/System.Management.Automation/engine/serialization.cs::DeserializingTypeConverter@L6670-L7678. +- **LSN-CLIXML-03** [REPLICATE] (area: NLD-SEC) Prohibit DTDs, cap entity expansion and nesting depth in every XML or structured reader. Evidence: src/System.Management.Automation/engine/serialization.cs::InternalDeserializer.GetXmlReaderSettingsForCliXml@L4172-L4191. +- **LSN-CLIXML-04** [REDESIGN] (area: NLD-SEC) Add an explicit maximum input size and a streaming limit to the primary reader, and mark sensitive properties so depth-1 bags cannot leak them. DEV: DEV-SER-215. Evidence: src/System.Management.Automation/engine/serialization.cs::InternalSerializer.GetDepthOfSerialization@L2322-L2381. +- **LSN-CLIXML-05** [REPLICATE] (area: NLD-TYP) Keep a marker on deserialized objects (type-name prefix and flag) so formatting and policy can tell data from live objects. Evidence: src/System.Management.Automation/engine/serialization.cs::Deserializer@L493-L819. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-068): `HandleComplexTypePSObject`, `HandleKnownContainerTypes` and enum handling, CIM instance serialization, `ReadListContainer`/`ReadDictionary` result types, `ReadProperties` details, type-name table (`TNRef`) handling, the `Deserialized.` helper functions (lines 682–820), `PSPrimitiveDictionary` restrictions, SecureString protection on Unix, script block deserialization default path. +- `UNVERIFIED` (VB-069): the cmdlets `Export-Clixml`, `Import-Clixml`, `ConvertTo-CliXml`-style entry points and the PSSerializer corner cases. + +Verification: `serialization.cs` lines 34–84, 86–220 (greps), 335–345, 971–1100, 2295–2388, 3656–3757, 4172–4215, 4792–4810, 4910–4975, 5167–5240, 6670–6740 and `SerializationStrings.cs` constants were read. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/05-remoting/transports.md b/docs-out/05-remoting/transports.md new file mode 100644 index 0000000..0047c4e --- /dev/null +++ b/docs-out/05-remoting/transports.md @@ -0,0 +1,188 @@ +--- +id: REM-TRANSPORT-001 +title: Remoting transports and remoting commands +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: OUT-OF-SCOPE-DOCUMENTED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs, src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs, src/System.Management.Automation/engine/remoting/fanin/WSManTransportManager.cs, src/System.Management.Automation/engine/remoting/fanin/WSManPlugin.cs, src/System.Management.Automation/engine/remoting/server/OutOfProcServerMediator.cs, src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs, src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs, src/System.Management.Automation/engine/remoting/commands/PSRemotingCmdlet.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/remoting/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json, data/conditional-compilation.json] +--- + +# Remoting transports and remoting commands + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `OUT-OF-SCOPE-DOCUMENTED`. Protocol layers above the transport: [psrp-protocol](psrp-protocol.md). The child-process job transport is described in [jobs-and-concurrency](../03-runtime/jobs-and-concurrency.md). + +## 1. Purpose and scope [SPEC] + +A transport carries PSRP fragments between a client engine and a server engine. This document covers the transport abstraction, the four concrete families — WSMan (HTTP/HTTPS, Windows-centric), SSH, named pipe / process, and Hyper-V socket (VM and container) — the connection-info classes the SDK uses to select them, the server-side mediators/plugins, and the user-facing remoting command flow (`Invoke-Command`, `New-PSSession`, `Enter-PSSession`). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs::BaseTransportManager@L194-L464 | abstract class | common transport-manager base (fragmentor, receive callbacks, constants) | B | +| src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs::BaseClientSessionTransportManager@L1088-L1171, BaseClientCommandTransportManager@L1173-L1283 | abstract classes | client session and per-command managers | B | +| src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs::AbstractServerSessionTransportManager@L1533-L1560 | abstract class | server session manager | B | +| src/System.Management.Automation/engine/remoting/fanin/WSManTransportManager.cs::WSManClientSessionTransportManager@L281-L2775, WSManClientCommandTransportManager@L2782-L4170 | classes | WSMan client over the native WSMan API | B | +| src/System.Management.Automation/engine/remoting/fanin/WSManNativeAPI.cs | classes | P/Invoke declarations for WSMan | B | +| src/System.Management.Automation/engine/remoting/fanin/WSManPlugin.cs, WSManPluginShellSession.cs, WSManPluginTransportManager.cs, WSManPluginFacade.cs | classes | WSMan server plugin (`pwrshplugin.dll` hosts the engine inside the WinRM service) | B | +| src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs | classes | out-of-process client managers (child process, SSH, named pipe, Hyper-V socket) and server managers | B | +| src/System.Management.Automation/engine/remoting/server/OutOfProcServerMediator.cs | classes | server-side loops: stdio, named pipe, Hyper-V socket | B | +| src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs::WSManConnectionInfo@L386-L1538, NewProcessConnectionInfo@L1545-L1704, NamedPipeConnectionInfo@L1705-L1927, SSHConnectionInfo@L1928-L2947, VMConnectionInfo@L2948-L3091, ContainerConnectionInfo@L3092-L3803 | classes | what the SDK/cmdlets pass to select a transport | B | +| src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs | classes | named-pipe client/server and pipe naming | B | +| src/System.Management.Automation/engine/remoting/commands/PSRemotingCmdlet.cs, InvokeCommandCommand.cs, newrunspacecommand.cs | cmdlets | remoting command surface | B | + +## 3. Responsibilities and non-responsibilities + +Owns: moving fragments, connection setup and teardown, timeouts, server-side session creation per transport, remoting cmdlet parameter sets. Does not own: message semantics (PSRP), authentication providers of the OS or SSH, endpoint configuration (JEA), job objects. + +## 4. Key types and contracts [IMPL] + +- **Transport manager contract**: a *session* manager per runspace pool and a *command* manager per pipeline; each exposes create, send data (fragments produced by the shared `Fragmentor`; default size 32 KB), receive callback (reassembly in the base class), close, and error events (`TransportErrorOccuredEventArgs`). Constants: connection open timeout 3 minutes, operation timeout 3 minutes, close timeout 1 minute, server keep-alive 4 minutes, maximum received object size 10 MB, idle timeout default "server's" (src/System.Management.Automation/engine/remoting/fanin/BaseTransportManager.cs@L209-L225). +- **Connection info classes** (all derive from `RunspaceConnectionInfo`, which carries `OpenTimeout`, `CancelTimeout`, `OperationTimeout`, `IdleTimeout`, credentials and culture): `WSManConnectionInfo` (URI, port, `AuthenticationMechanism`, SSL options, proxy, `IncludePortInSPN`, shell URI/configuration name), `NewProcessConnectionInfo` (internal: start a child `pwsh` with `-s`), `NamedPipeConnectionInfo` (process id or pipe name, app domain), `SSHConnectionInfo` (user, host, key file, port, **subsystem default `powershell`**, connecting timeout default infinite, extra `-o` options), `VMConnectionInfo` (Hyper-V VM id), `ContainerConnectionInfo` (container id; Windows containers over Hyper-V sockets or named pipes). `AuthenticationMechanism`: `Default`, `Basic`, `Negotiate`, `NegotiateWithImplicitCredential`, `Credssp`, `Digest`, `Kerberos`. +- **WSMan**: the client calls the native WSMan client API (declared in `WSManNativeAPI.cs`); the server side is a native plugin (`pwrshplugin.dll`, `PSPluginDLLName`) loaded by the WinRM service (`INFERENCE` from the class names and the DLL constant), with managed entry wrappers (`WSManPluginManagedEntryWrapper`) and per-shell/per-command session classes. On Unix the client explicitly **disallows Basic authentication over plain HTTP** (`BasicAuthOverHttpNotSupported`); WSMan server hosting is Windows-only (`UNVERIFIED` full conditional-compilation picture; 4 and 6 `#if UNIX` regions in the native API and client files). +- **SSH**: the client starts the local `ssh` executable with the target and the arguments `-s ` (default `powershell`) plus optional `-o key=value` options (other arguments such as port, key file and user are properties of the connection info; how they map to `ssh` arguments was not read, `UNVERIFIED`); it talks to the child's stdin/stdout/stderr using the out-of-process packet protocol; the **server** must define a `Subsystem powershell … pwsh -SSHServerMode -NoLogo -NoProfile` line in `sshd_config` (source comment gives the three example lines). `pwsh -sshs` runs `StdIOProcessMediator` with `combineErrOutStream: true` (src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs::SSHConnectionInfo.StartSSHProcess@L2196-L2342). +- **Out-of-process packet protocol** (used by child process, SSH, named pipe, Hyper-V socket): newline-delimited XML elements — `base64 fragment` with acknowledgement ``, `` and `` (create a pipeline transport), `` and ``, `` and `` (stop); the server/reader parses each element with an XML reader configured for fragments with comments and processing instructions ignored and no resolver; any text node outside an element is a transport error; a `Data` element must have exactly the two attributes (src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs::OutOfProcessUtils@L37-L403). `DataPriorityType`: `Default = 0`, `PromptResponse = 1`. +- **Named pipe**: pipe name `PSHost....` (the default app domain name is used by the host process); on Unix the .NET runtime prefixes the file name with `CoreFxPipe_`, so discovery searches `CoreFxPipe_PSHost*`. Windows server pipe: message mode, **remote clients rejected**, one instance, a DACL granting full control to Administrators and, if the process is not elevated, to the current user; Unix: `CurrentUserOnly` (src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs@L29-L125, `GetServerPipeSecurity@L651-L700`, `CreateNamedPipe@L426-L492`). `pwsh -NamedPipeServerMode` runs `RemoteSessionNamedPipeServer.RunServerMode`; `-CustomPipeName` creates an additional named-pipe listener in an interactive host (see [console-host](../04-hosting/console-host.md)). +- **Hyper-V sockets** (VM and container, Windows): `HyperVSocketMediator` on the server with a token and UTC time stamp (`-v2socketservermode`) and client managers for VM and container (src/System.Management.Automation/engine/remoting/common/RemoteSessionHyperVSocket.cs, `UNVERIFIED` details). +- **Remoting commands** (`PSRemotingCmdlet.cs` parameter-set constants): target selection sets `ComputerName`, `Uri`, `Session` (existing PSSession), `SSHHost`, `SSHHostHashParam`, `ContainerId`, `VMId`, `VMName`, `UseWindowsPowerShellParameterSet` (Windows PowerShell compatibility), plus `FilePath…` variants for `-FilePath`; `Invoke-Command` runs the script block (or file) on every target in parallel up to a throttle limit; `New-PSSession` creates runspaces (kept in the runspace repository); `Enter-PSSession` pushes a remote runspace onto the console host (`IHostSupportsInteractiveSession.PushRunspace`) so the prompt is remote until `Exit-PSSession`; `Receive-PSSession`/`Connect-PSSession`/`Disconnect-PSSession` manage disconnected sessions; job cmdlets use `PSRemotingJob` ([jobs-and-concurrency](../03-runtime/jobs-and-concurrency.md)). + +## 5. Data model + +Client: connection info → transport manager (one per pool, one per command) → `Fragmentor` and priority send collection → bytes. Server: transport manager per session/command, `ServerRemoteSession`, runspace pool driver, per-pipeline drivers. Out-of-process: text writer/reader wrapping stdin/stdout (or pipe/socket) with base64 payloads. + +## 6. Control flow + +```mermaid +sequenceDiagram + participant App as Cmdlet or SDK + participant CI as ConnectionInfo + participant TM as Client transport manager + participant W as Wire (WSMan, ssh stdio, pipe, socket) + participant M as Server mediator or plugin + participant S as ServerRemoteSession + App->>CI: choose transport (ComputerName, SSHHost, pipe, VM, container) + CI->>TM: create session transport manager + TM->>W: connect (ssh child process, pipe connect, WSMan CreateShell) + W->>M: server loop starts (sshd subsystem, pipe server, WinRM plugin) + M->>S: create session (configuration name or file) + TM->>W: Data packets (SESSION_CAPABILITY ... ) + W->>S: reassembled messages (PSRP) + TM->>W: Command packet per pipeline + TM->>W: Close / Signal on stop +``` + +## 7. Algorithms + +Not applicable beyond §4 (packet grammar, pipe naming, SSH argument construction). + +## 8. Observable behavior [SPEC] + +- `New-PSSession -HostName h -UserName u` (SSH) starts a local `ssh -s powershell` and talks PSRP over its stdio; `-ComputerName` uses WSMan; `Enter-PSSession -ProcessId n` connects to a local process by named pipe. +- Output of a remote command carries `PSComputerName`; a timeout or broken connection raises `PSRemotingTransportException` and the session goes to `Broken`/`Disconnected`. +- Basic auth over HTTP is refused on Unix clients. +- Tests: §16 (many remoting tests require a configured endpoint). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`PSRemotingTransportException` (with error ids such as `ConnectFailed`, `IPCUnknownNodeType`, `IPCWrongAttributeCountForDataElement`, `IPCInsufficientDataforElement`, `IPCOnlyTextExpectedInDataElement`) for connection and parsing faults; timeouts raise transport errors and close; SSH process exit before negotiation is a connection failure; named pipe creation failure reports the Win32 error (`CannotCreateNamedPipe`); invalid port numbers are rejected (`PortIsOutOfRange`, range 0–65535); Windows-only transports raise platform errors on Unix. + +## 10. Concurrency and threading + +Reads of stdout/stderr from the SSH child run on dedicated reader threads that parse packets and post to the session state machine; writes are serialized by `OutOfProcessTextWriter`; WSMan callbacks come from native threads and are marshalled; the client throttle manager caps concurrent operations; server loops are single-threaded readers feeding drivers that run pipelines on runspace threads. + +## 11. Extension points and customization hooks + +Subclass `RunspaceConnectionInfo` / transport managers (public abstract `ClientSessionTransportManagerBase`); `PSSessionOption` (timeouts, culture, proxy, size limits); `sshd_config` subsystem entry; WSMan endpoint registration; session configuration name/file passed to `-ConfigurationName`/`-ConfigurationFile`; `$PSSenderInfo` and `PSSessionConfiguration` (see [jea](jea.md)). + +## 12. Dependencies + +Inbound: remoting cmdlets, `RunspaceFactory`, jobs. Outbound: `Fragmentor`/CLIXML, OS `ssh`, WinRM and the WSMan native API (Windows), .NET named pipes, Hyper-V sockets, process APIs. + +## 13. Performance notes + +Out-of-process transports Base64-encode every fragment (about a third of overhead) and acknowledge data packets; WSMan adds HTTP framing; startup of SSH or process transports costs a process start; named pipes avoid network stack latency; fragment size (32 KB) bounds memory per send. + +## 14. Security considerations + +Security properties come from the transport: WSMan — Kerberos/Negotiate message-level encryption or HTTPS (Basic and Digest are weaker; Basic-over-HTTP blocked on Unix); SSH — host keys and user auth of OpenSSH, with the server subsystem running as the authenticated user; named pipe — local only, ACL'd to administrators and the creating user (Windows) or current user (Unix); Hyper-V socket — token + timestamp. The protocol relies on this since version 2.4. `-CustomPipeName` and `-NamedPipeServerMode` create listeners that must be protected by the pipe ACL. Server side, the configured endpoint decides the initial session state (language mode, visibility); command-line server modes can be disabled with the file-only-entry policy. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| securestring | 4 | +| jea | 2 | +| remoting-deser | 1 | +| **total** | 7 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/engine/remoting/fanin/WSManTransportManager.cs` | 4 | +| `src/System.Management.Automation/engine/remoting/fanin/WSManNativeAPI.cs` | 2 | +| `src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs` | 1 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here (if remoting is ever built): only authenticated, encrypted transports are offered; local transports are bound to the user and refuse remote peers; listeners are off by default and enabled by policy; every connection is logged with peer identity before the runspace is created. + +## 15. Enterprise considerations + +Security first: §14. Observability: ETW remoting tracing, transport error events, session/command GUIDs, timeouts surfaced to the caller. Supportability: multiple transports let admins choose; SSH works across platforms; WSMan integrates with Windows management. Testability: out-of-process transports can be tested with child processes; WSMan needs an endpoint. Resource limits: throttle limit per `Invoke-Command`, per-session timeouts, received-object size cap. Missing: a transport-neutral authentication layer, mutual TLS option for PSRP-over-socket, rate limiting per peer. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/SSHRemoting/SSHRemoting.Basic.Tests.ps1` | 1 | 2 | 19 | +| `test/powershell/engine/Remoting/CustomConnection.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/Remoting/InvokeCommandRemoteDebug.Tests.ps1` | 1 | 0 | 4 | +| `test/powershell/engine/Remoting/PSSession.Tests.ps1` | 2 | 0 | 2 | +| `test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1` | 5 | 0 | 19 | +| `test/powershell/engine/Remoting/RemoteSession.Disconnect.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/engine/Remoting/RoleCapabilityFiles.Tests.ps1` | 1 | 0 | 3 | +| `test/powershell/engine/Remoting/RunspacePool.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/engine/Remoting/SSHRemotingAPI.Tests.ps1` | 1 | 1 | 3 | +| `test/powershell/engine/Remoting/SSHRemotingCmdlets.Tests.ps1` | 4 | 0 | 4 | +| `test/powershell/engine/Remoting/SessionOption.Tests.ps1` | 1 | 0 | 4 | +| **total (11 files)** | 19 | 3 | 63 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_NamedPipe.cs` | 2 | 0 | `NamedPipeTests` | +| `test/xUnit/csharp/test_RemoteHyperV.cs` | 0 | 0 | `RemoteHyperVTests`, `of` | + +Matching is by path keyword (`APPROX`): `Remoting/`, `engine/Remoting`, `SSHRemoting`, `test_NamedPipe`, `test_RemoteHyperV`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: multiple transports share the same fragment/message layers, so the engine is transport-agnostic. +- `FACT-SRC`: SSH remoting reuses OpenSSH as the authenticated, encrypted pipe and runs the engine as a subsystem (no new listener). +- `INFERENCE`: Base64 text framing on stdio makes the protocol robust to stream corruption and NUL bytes at a size cost. +- `FACT-SRC`: named-pipe transport is limited to the local machine and the same user (or administrators). + +## 18. Guidance for the new shell [LESSON] + +- **LSN-TRANS-01** [AVOID] (area: NLD-SEC) Do not implement WSMan, Hyper-V socket or container transports; they are Windows-specific and tied to native components. DEV: DEV-REM-211. Evidence: src/System.Management.Automation/engine/remoting/fanin/WSManNativeAPI.cs@L1-L3105. +- **LSN-TRANS-02** [ADAPT] (area: NLD-SEC) If remoting is added, start with an SSH-subsystem or local-pipe transport that delegates authentication and encryption to an existing secure channel. Evidence: src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs::SSHConnectionInfo@L1928-L2942. +- **LSN-TRANS-03** [REPLICATE] (area: NLD-SEC) Local pipes bound to the creating user, single instance, remote clients rejected. Evidence: src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs::NamedPipeUtils@L25-L156. +- **LSN-TRANS-04** [ADAPT] (area: NLD-RUN) Use length-prefixed binary frames instead of Base64 XML elements on stdio, keeping acknowledgement and signal semantics. Evidence: src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs::OutOfProcessUtils@L37-L403. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-066): `WSManTransportManager` and `WSManPlugin*` bodies, `WSManNativeAPI`, Hyper-V socket classes, `ContainerProcess`, container/VM connection details, `PSRemotingCmdlet` and `InvokeCommandCommand` bodies (throttling, argument passing), `newrunspacecommand`, session option defaults. +- `UNVERIFIED` (VB-067): the exact `ssh` argument list order in `StartSSHProcess` (only the `-o`, `-s ` parts were read), `OutOfProcessMediatorBase` read loop, `NewProcessConnectionInfo`. + +Verification: `OutOfProcTransportManager.cs` lines 37–260, `RunspaceConnectionInfo.cs` class list, constants and lines 1928–2070 and 2193–2341 (partly), `RemoteSessionNamedPipe.cs` lines 29–125 (grep) and 426–500, 651–700, `BaseTransportManager.cs` lines 194–225 (constants), `PSRemotingCmdlet.cs` parameter-set constants (grep), class lists of the fanin and server directories were read. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/06-platform/native-interop.md b/docs-out/06-platform/native-interop.md new file mode 100644 index 0000000..6aa65d3 --- /dev/null +++ b/docs-out/06-platform/native-interop.md @@ -0,0 +1,157 @@ +--- +id: PLAT-NATIVE-001 +title: Native interop and the native libraries +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/CoreCLR/CorePsPlatform.cs, src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs, src/System.Management.Automation/utils/PInvokeDllNames.cs, src/System.Management.Automation/utils/PlatformInvokes.cs, src/System.Management.Automation/engine/Interop/Windows/CoInitializeEx.cs, src/System.Management.Automation/engine/Interop/Windows/CreateHardLink.cs, src/System.Management.Automation/engine/Interop/Windows/FindFirstFile.cs, src/System.Management.Automation/utils/tracing/SysLogProvider.cs, src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs, src/System.Management.Automation/System.Management.Automation.csproj, src/libpsl-native/README.md, src/powershell-native/README.md, docs/building/internals.md] +mirror_cards: [src-mirror/src/System.Management.Automation/CoreCLR/_folder.md, src-mirror/src/System.Management.Automation/utils/_folder.md] +generated_from: [data/types.json, data/tests.json, data/enforcement-points.json] +--- + +# Native interop and the native libraries + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. The OS split that decides which native layer is compiled: [windows-unix-split](windows-unix-split.md). Managed assembly loading: [modules](../03-runtime/modules.md). + +## 1. Purpose and scope [SPEC] + +The engine reaches the OS through three kinds of native code: (1) the operating system's own libraries (Win32 API sets on Windows), (2) a small C library `libpsl-native` on Linux and macOS, and (3) Windows-only native components that ship as a package (WinRM plugin, ETW provider, `Microsoft.PowerShell.Native`). This document describes how each is declared, marshalled, loaded and tested, and where the sources live. The native C source is **not in this repository** (see §4). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122 | internal static partial class | 17 `LibraryImport` entries into `libpsl-native` (Unix) | B | +| src/System.Management.Automation/engine/Interop/Windows/ | folder (33 files) | one small file per Win32 function, `Interop.Windows` partial class, source-generated marshalling | B | +| src/System.Management.Automation/utils/PInvokeDllNames.cs::PinvokeDllNames@L13-L140 | internal static class | 125 string constants naming API-set DLLs | B | +| src/System.Management.Automation/utils/PlatformInvokes.cs::PlatformInvokes@L11-L678 | internal static class | older `DllImport`-era structs, enums and safe handles (FILETIME, STARTUPINFO, privileges) | B | +| src/System.Management.Automation/utils/tracing/SysLogProvider.cs::SysLogProvider.NativeMethods@L370-L532 | nested class | `DllImport` of syslog wrappers (`Native_SysLog`, `Native_OpenLog`, `Native_CloseLog`) | B | +| src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext@L19-L565 | internal sealed class | last-resort native library resolver | B | +| src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs::SSHConnectionInfo@L1928-L2942 | public sealed class | Unix `ForkAndExecProcess` for the SSH child process | B | +| src/libpsl-native/README.md, src/powershell-native/README.md | readme | state that the code moved to the PowerShell-native repository | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the declaration style, naming of native entry points, the pinned native package, the native library search rule, and the test hooks. Does not own: the C source of `libpsl-native` or the Windows native components (separate repository; this tree holds only two notices), the behavior built on top of the calls (see the subsystem documents), or WSMan protocol details ([transports](../05-remoting/transports.md)). + +## 4. Key types and contracts [IMPL] + +- **Where the native code lives.** `src/libpsl-native/` and `src/powershell-native/` contain only a README each; both say the code is being migrated to the `PowerShell-native` repository and that new pull requests go there. The engine takes the binaries from the NuGet package `Microsoft.PowerShell.Native` version `700.0.2` (src/System.Management.Automation/System.Management.Automation.csproj@L44-L46). `docs/building/internals.md` names `libpsl-native.so/.dylib` as the Unix dependency and `pwrshplugin.dll` as the WinRM plugin and says they are built once and consumed as packages. The package contents and the C source are `UNVERIFIED` (VB-074). +- **Unix: `libpsl-native`.** The class `Platform.Unix.NativeMethods` declares `private const string psLib = "libpsl-native"` and 17 `LibraryImport` entries: `GetErrorCategory`, `GetPPid`, `GetLinkCount`, `IsExecutable`, `GetCurrentThreadId`, `KillProcess`, `WaitPid`, `SetDate`, `CreateSymLink`, `CreateHardLink`, `GetUserFromPid`, `IsSameFileSystemItem`, `GetInodeData`, `GetCommonLStat`, `GetCommonStat`, `GetPwUid`, `GetGrGid`. String marshalling is UTF-8 (`StringMarshalling.Utf8`); the file comment says "Ansi is a misnomer, it is hardcoded to UTF-8 on Linux and macOS". C `bool` is one byte, so returns are marked `UnmanagedType.I1` where the native side returns a C `bool` and `UnmanagedType.Bool` for `KillProcess`/`WaitPid` (a mixed convention in the same class; the native header decides, `UNVERIFIED`). Calls that report errors use `SetLastError = true` (`GetLinkCount`, `SetDate`, `GetCommonLStat`, `GetCommonStat`) (src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122). +- **Struct contracts with C.** `UnixTm` (9 ints, `Pack = 4`) mirrors `private_tm` in the native `setdate.h`; `CommonStatStruct` presents "each member of the stat structure as the largest type of that member across all stat structures on the platforms we support" (22 fields: inode, mode, ids, counts, sizes, three times, block size, device, blocks, seven item-kind flags and three mode flags). These layouts are a hand-kept ABI: there is no generated header. A changelog entry (6.1) records a memory-corruption bug in `SetDate` caused by exactly this kind of mismatch (CHANGELOG/6.1.md line 433). +- **Syslog and process spawn.** Syslog wrappers are `DllImport` with `CharSet.Ansi` and explicit `EntryPoint` names (`Native_SysLog`, `Native_OpenLog`, `Native_CloseLog`). SSH child-process creation on Unix calls `ForkAndExecProcess` (`DllImport("libpsl-native", CharSet.Ansi, SetLastError = true)`) with null-terminated `argv`/`envp` arrays built by hand in unmanaged memory and freed in a `finally` (src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs@L2355-L2600). Hence **two declaration styles** exist on the Unix side: source-generated `LibraryImport` (new) and runtime-marshalled `DllImport` (older). +- **Windows: API sets and per-function files.** New Win32 declarations are one file each under `engine/Interop/Windows/` (33 files such as `CreateHardLink.cs`, `FindFirstFile.cs`, `CoInitializeEx.cs`, `GetOEMCP.cs`, `NetShareEnum.cs`, `WNetAddConnection2.cs`). Each is `#nullable enable`, in `internal static partial class Interop.Windows`, uses `[LibraryImport("api-ms-win-…dll", EntryPoint = "…W", StringMarshalling = Utf16, SetLastError = true)]` and, where a handle is returned, a `SafeHandle` subclass (`SafeFindHandle` in `FindFirstFile.cs`). The DLL names are API-set contracts (`api-ms-win-core-file-l2-1-0.dll`), not `kernel32.dll`, so the code binds to the OneCore surface. 51 of those files set `SetLastError` (grep count, APPROX). The 125 constants in `PinvokeDllNames` hold the same API-set names for the older `DllImport` declarations (note its numbering comments skip `/*29*/`) (src/System.Management.Automation/utils/PInvokeDllNames.cs::PinvokeDllNames@L13-L140). +- **Counts (APPROX, grep).** `DllImport(` 282 and `LibraryImport(` 78 occurrences in 66 files under `src/`. Biggest `DllImport` files: `Microsoft.PowerShell.CoreCLR.Eventing/.../UnsafeNativeMethods.cs` (46), `security/nativeMethods.cs` (44), `ConsoleControl.cs` (32), `WSManNativeAPI.cs` (26), `PdhHelper.cs` (23), `security/Win32Native/WinTrust.cs` (18), `PlatformInvokes.cs` (14). Of the `LibraryImport` entries, 32 are under `System.Management.Automation/engine` (the Interop folder). +- **Native library search.** The engine registers `AssemblyLoadContext.Default.ResolvingUnmanagedDll += NativeDllHandler` as a "last resort" (the comment lists the default order: `DllImportResolver` callbacks, then `LoadUnmanagedDll`, then this event). `NativeDllHandler(Assembly, libraryName)` builds `//` with `rid` = `win-`, `linux-` or `osx-` plus the lower-case process architecture and `ext` = `.dll`, `.so`, `.dylib`, and loads it with `NativeLibrary.TryLoad`; failure returns `IntPtr.Zero` (src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext.NativeDllHandler@L243-L250 and GetNativeDllSubFolderName@L539-L563). This rule lets a module ship per-RID native libraries beside its managed assembly. The default probing (app directory) is what finds `libpsl-native` itself (`UNVERIFIED`, VB-074). +- **Native assembly loading from memory.** `PowerShellUnsafeAssemblyLoad.LoadAssemblyFromNativeMemory` is an `UnmanagedCallersOnly` entry that lets a native host hand a managed assembly image to the engine; the xUnit test calls it through an `unmanaged` function pointer (src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellUnsafeAssemblyLoad@L596-L625; test: test/xUnit/csharp/test_NativeInterop.cs). + +## 5. Data model + +Marshalled structures: `UnixTm`, `CommonStatStruct` (Unix); `FILETIME`, `STARTUPINFO`, `PROCESS_INFORMATION`, `SECURITY_ATTRIBUTES`, `TOKEN_PRIVILEGE`, `LUID`, `PRIVILEGE_SET` and flag enums (Windows, `PlatformInvokes.cs`). Safe handles: `SafeFindHandle`, `SafeLocalMemHandle`. Strings are UTF-8 (Unix) or UTF-16 (Windows, `…W` entry points). + +## 6. Control flow + +```mermaid +flowchart LR + C[managed caller] --> S{OS family} + S -->|Windows| I[Interop.Windows LibraryImport to api-ms-win DLL] + S -->|Unix| N[Platform.Unix.NativeMethods LibraryImport to libpsl-native] + N --> R{found by default probing?} + R -->|no| H[NativeDllHandler: folder/rid/name.ext] + I --> OS[(OS kernel)] + N --> OS +``` + +## 7. Algorithms + +Native resolver path building (§4). Stat conversion: the native call fills `CommonStatStruct`; the managed side copies fields, converts seconds since epoch to local `DateTime` with an exception fallback, and chooses `ItemType` by the first flag equal to 1 in the order directory, file, symbolic link, block device, character device, named pipe, otherwise socket ([windows-unix-split](windows-unix-split.md) §4). + +## 8. Observable behavior [SPEC] + +A missing or incompatible `libpsl-native` makes the first Unix call that needs it throw a load error (`DllNotFoundException` or `EntryPointNotFoundException`, `INFERENCE` from the .NET rules); the hosting test `test_HostingBasic.cs` notes that `New-Item -ItemType SymbolicLink` "uses libpsl-native, hence" serves to validate native dependencies. `Assembly.LoadNative.Tests.ps1` builds `/testDllNativeFolder//nativedll.` from a runtime host library and checks that the resolver finds it. + +## 9. Error handling and edge cases + +- Unix calls that fail return non-zero and the managed code reads `Marshal.GetLastWin32Error()` (works for `errno` on Unix) and throws `Win32Exception`; `GetErrorCategory` maps `errno` to `ErrorCategory` in native code. +- `ForkAndExecProcess` failure after the arrays are built is converted to an exception; the arrays are freed on all paths (`finally`). +- `GetNativeDllSubFolderName` returns an empty string on an unknown OS family (for example FreeBSD); the resolver would then probe `/` with an empty extension (`INFERENCE`). +- On a Windows build, Unix `NativeMethods` is not compiled; on a Unix build, `Interop.Windows` is removed. Both are compile-time facts ([windows-unix-split](windows-unix-split.md) §4). + +## 10. Concurrency and threading + +The native calls are stateless wrappers. Static caches around them are not synchronized ([windows-unix-split](windows-unix-split.md) §10). `GetCurrentThreadId` returns the OS thread id so ETW and syslog entries correlate with OS tools. + +## 11. Extension points and customization hooks + +Modules can ship per-RID native libraries under `//` and rely on the resolver. The engine itself offers no plug-in point for replacing `libpsl-native`. + +## 12. Dependencies + +Inbound: file system provider (links, stat, hard-link count), native command processor (executable test, kill, wait), tracing (syslog), SSH transport (process spawn), formatting (user and group names), `Set-Date`. Outbound: .NET interop marshaller, the NuGet package `Microsoft.PowerShell.Native`, the OS C library. + +## 13. Performance notes + +Source-generated `LibraryImport` avoids runtime IL stub generation and is trim-friendly (`INFERENCE` from the generator's purpose; no measurement). The stat path makes one native call per item plus lazy name lookups with managed caches ([windows-unix-split](windows-unix-split.md) §13). + +## 14. Security considerations + +- Every native boundary is a place where managed safety checks end: hand-built `argv`/`envp` arrays, fixed struct layouts, `SetDate` writing system time, `KillProcess(pid)`. The engine passes caller-chosen strings (paths, command lines) across it. +- `ForkAndExecProcess` receives `filename`, `argv`, `envp` and a working directory. The managed side builds `envp` from `ProcessStartInfo.Environment` (`CopyEnvVariables`), so the child sees the full parent environment unless the caller changes it. +- The last-resort resolver loads any `/.` found beside the calling assembly. Anyone who can write to a module folder can place a native library there (`INFERENCE`); this relies on file-system permissions and signing, not on the resolver. +- No Windows or Unix native call in the files read checks the language mode: enforcement must happen before the call ([Phase 5](../README.md)). + +### Security surface + + +No scanner seed hit under the prefixes of this subsystem (`UNVERIFIED`: absence of a hit is not proof of absence of a check). + + +What the new shell must guarantee here: one reviewed, versioned native surface per OS with generated bindings and layout checks; a native library search rule that does not load from writable locations by default; no caller-controlled string reaches a process-spawn or time-setting call without a policy check. + +## 15. Enterprise considerations + +Supportability: the Unix half depends on a binary that must match the OS and architecture (`linux-*`, `osx-*` packages); a container image without it breaks links, stat and process control. The native source being in a separate repository splits review and release. Observability: syslog entries carry a commit hash and thread id (see `SysLogProvider` header). Packaging: `tools/packaging/boms/linux.json` and `mac.json` list `libpsl-native.so/.dylib` as required files and `build.psm1` lists the same file as a required dependency for xUnit runs on Unix. + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/engine/Basic/Assembly.LoadNative.Tests.ps1` | 1 | 0 | 1 | +| `test/powershell/engine/Basic/NativeCommandBytePiping.Tests.ps1` | 1 | 0 | 8 | +| **total (2 files)** | 2 | 0 | 9 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_CorePsPlatform.cs` | 12 | 0 | `PlatformTests` | +| `test/xUnit/csharp/test_NativeInterop.cs` | 1 | 0 | `NativeInterop`, `Utt` | + +Matching is by path keyword (`APPROX`): `Assembly.LoadNative`, `NativeCommandBytePiping`, `test_NativeInterop`, `test_CorePsPlatform`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: a small native library supplies POSIX calls .NET does not expose (link count, inode, `setdate`, user and group names, `errno` mapping); the alternative of calling libc directly was not taken (the reason is not recorded in the files read). +- `FACT-SRC`: new Windows bindings are split one function per file with source generation; old bindings remain in `PlatformInvokes.cs` and large `DllImport` files, so the code base carries three generations of interop style. +- `FACT-SRC`: native code moved out of the main repository and arrives as a package, which speeds builds ("the native component code seldom changes", `docs/building/internals.md`) at the cost of hand-kept struct layouts on the managed side. +- `INFERENCE`: API-set DLL names keep the Windows build portable across SKUs (Nano, IoT, desktop), which fits the headless-SKU queries in [windows-unix-split](windows-unix-split.md). + +## 18. Guidance for the new shell [LESSON] + +- **LSN-NAT-01** [REDESIGN] (area: NLD-PLAT) Keep one small native surface per OS with generated bindings and a layout check in CI, instead of hand-copied structs. Evidence: src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122. DEV: DEV-PLAT-231. +- **LSN-NAT-02** [REPLICATE] (area: NLD-PLAT) One file per native function, source-generated marshalling, `SafeHandle` for owned handles, UTF-16 on Windows and UTF-8 on Unix. Evidence: src/System.Management.Automation/engine/Interop/Windows/FindFirstFile.cs. +- **LSN-NAT-03** [ADAPT] (area: NLD-PLAT) Pick one interop style (source-generated) and one `bool` marshalling convention per class; do not let three generations coexist. Evidence: src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122. +- **LSN-NAT-04** [REDESIGN] (area: NLD-SEC) Native library resolution must prefer the application directory and a signed or pinned per-RID folder, and refuse writable locations unless a policy allows them. DEV: DEV-PLAT-232. Evidence: src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext.NativeDllHandler@L243-L250. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-074): the C source and exported symbol list of `libpsl-native` (not in this tree); the contents of the `Microsoft.PowerShell.Native` package; how default probing finds `libpsl-native` in the publish layout; the Windows native components (`pwrshplugin.dll`, ETW provider) beyond the names in `docs/building/internals.md`. +- `UNVERIFIED` (VB-075): the bodies of the 33 `Interop/Windows` files other than the three sampled (`CoInitializeEx.cs`, `CreateHardLink.cs`, `FindFirstFile.cs` header), `security/nativeMethods.cs`, `WinTrust.cs`, `ConsoleControl.cs`, `WSManNativeAPI.cs`, `PdhHelper.cs`, `UnsafeNativeMethods.cs`. +- `UNVERIFIED`: whether `ForkAndExecProcess` is used by anything besides the SSH transport; the exact end of the Unix process-creation region. + +Verification: `CorePsPlatform.cs` lines 1–1125 read in full; `PInvokeDllNames.cs` read in full; `PlatformInvokes.cs` only the type list; `CorePsAssemblyLoadContext.cs` lines 98–108, 205–260, 300–370, 539–566 read; `RunspaceConnectionInfo.cs` lines 2355, 2401, 2490–2525, 2585–2600 read; `SysLogProvider.cs` header and `NativeMethods` entry names via grep; `CoInitializeEx.cs`, `CreateHardLink.cs` and the head of `FindFirstFile.cs` read; the two README stubs and `docs/building/internals.md` lines 100–125 read; `test_NativeInterop.cs` lines 10–60 and `Assembly.LoadNative.Tests.ps1` lines 1–40 read. Counts are from `grep` over `src/`. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/06-platform/windows-unix-split.md b/docs-out/06-platform/windows-unix-split.md new file mode 100644 index 0000000..036914d --- /dev/null +++ b/docs-out/06-platform/windows-unix-split.md @@ -0,0 +1,203 @@ +--- +id: PLAT-SPLIT-001 +title: Windows and Unix platform split +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/CoreCLR/CorePsPlatform.cs, PowerShell.Common.props, src/System.Management.Automation/System.Management.Automation.csproj, src/System.Management.Automation/engine/NativeCommandProcessor.cs, src/System.Management.Automation/namespaces/FileSystemProvider.cs, src/System.Management.Automation/security/SecurityManager.cs, src/System.Management.Automation/security/SecuritySupport.cs, src/System.Management.Automation/engine/PSConfiguration.cs, src/System.Management.Automation/utils/PsUtils.cs, src/System.Management.Automation/utils/ClrFacade.cs, src/System.Management.Automation/engine/CommandSearcher.cs, src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs, src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs, docs-out/data/conditional-compilation.json] +mirror_cards: [src-mirror/src/System.Management.Automation/CoreCLR/_folder.md, src-mirror/src/System.Management.Automation/utils/_folder.md] +generated_from: [data/types.json, data/conditional-compilation.json, data/tests.json, data/enforcement-points.json] +--- + +# Windows and Unix platform split + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`. The native layer behind the Unix half: [native-interop](native-interop.md). Hosting differences: [console-host](../04-hosting/console-host.md). Native command launch: [command-discovery-and-invocation](../03-runtime/command-discovery-and-invocation.md). + +## 1. Purpose and scope [SPEC] + +PowerShell 7 runs on Windows, Linux and macOS from one code base. This document records *how* the code base is divided: which decisions are made at build time, which at run time, what the `Platform` class offers, and which engine behaviors differ per OS. It is the input for the new shell's decision on how to isolate OS differences (LSN-PLAT-01 to LSN-PLAT-04). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform@L16-L1124 | public static partial class | runtime platform queries, directories, `NonWindows*` shims | B | +| src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix@L507-L1123 | internal static partial class | stat model, `/proc` parent lookup, native method table | B | +| src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::XDG_Type@L298-L312 | public enum | directory kinds for XDG lookup | B | +| PowerShell.Common.props@L164-L172 | build props | defines `CORECLR`; defines `UNIX` when `IsWindows` is not true | B | +| src/System.Management.Automation/System.Management.Automation.csproj@L61-L76 | build file | removes Windows-only or Unix-only source files per OS | B | +| data/conditional-compilation.json | generated data | files and types that use each preprocessor symbol | B | +| src/System.Management.Automation/engine/NativeCommandProcessor.cs | class | native command launch, file-association fallback, exit-code text | B | +| src/System.Management.Automation/namespaces/FileSystemProvider.cs | class | links, name rules, open-with-default-app, mapped drives | B | +| src/System.Management.Automation/security/SecurityManager.cs, security/SecuritySupport.cs | classes | origin (zone) check, execution policy store | B | +| src/System.Management.Automation/engine/PSConfiguration.cs | class | `powershell.config.json`; syslog identity and keywords on Unix | B | +| src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext | internal sealed class | managed and native load rules; GAC fallback only on Windows | B | + +## 3. Responsibilities and non-responsibilities + +Owns: the two-level split (preprocessor symbol and runtime query), the platform class, the per-OS directory rules, and the places where behavior is intentionally different. Does not own: the P/Invoke declarations and the native library ([native-interop](native-interop.md)); WSMan and SSH transport differences ([transports](../05-remoting/transports.md)); language-mode and policy enforcement ([Phase 5](../README.md)). + +## 4. Key types and contracts [IMPL] + +- **Two switches, not one.** (1) *Build time*: `PowerShell.Common.props` defines `CORECLR` for every build and adds `UNIX` when `IsWindows` is not `true`; `IsWindows` is true if set to true or if `$(OS)` is `Windows_NT` (PowerShell.Common.props@L164-L172). The engine assembly is therefore **built per OS family**: a Linux assembly does not contain the Windows branches, and a Windows assembly does not contain the Unix branches. (2) *Run time*: `Platform.IsLinux`, `IsMacOS`, `IsWindows` forward to `OperatingSystem.IsLinux/IsMacOS/IsWindows`; `IsCoreCLR` is the constant `true` (src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform@L16-L1124). +- **Counts (APPROX, grep and generated data).** `data/conditional-compilation.json` lists the `UNIX` symbol in 122 files and 247 types. A line count of `#if UNIX` and `#if !UNIX` under `src/` gives 390 directives in 121 files. `Platform.IsWindows` appears 39 times in 24 files. The attributes `DllImport(` (282) and `LibraryImport(` (78) appear in 66 files (see [native-interop](native-interop.md)). +- **Source removal per OS.** On non-Windows builds the project removes `engine\COM`, `engine\ComInterop`, `ExtraAdapter.cs`, `engine\Interop\Windows`, `ManagementObjectAdapter.cs` and five remoting command files (`ConnectPSSession.cs`, `CustomShellCommands.cs`, `DisconnectPSSession.cs`, `ReceivePSSession.cs`, `TestPSSessionConfigurationFile.cs`). On Windows builds it removes `engine\Interop\Unix`; **no such directory exists in this tree** (only `engine/Interop/Windows`, 33 files), so that line is a leftover or a reserved path (`INFERENCE`). Windows builds also reference the project `Microsoft.PowerShell.CoreCLR.Eventing` (src/System.Management.Automation/System.Management.Automation.csproj@L25-L27 and @L61-L76). Consequence: `Register-PSSessionConfiguration` and the other session-configuration registration cmdlets (see [jea](../05-remoting/jea.md)) **do not exist on Unix builds**; Unix remoting uses SSH. +- **Windows SKU queries.** `IsNanoServer`, `IsIoT`, `IsWindowsDesktop` read the registry (`...\Server\ServerLevels` value `NanoServer`; `...\CurrentVersion` value `ProductName` equal to `IoTUAP`) once and cache the result; on Unix they return `false`. `IsWindowsDesktop` is `!IsNanoServer && !IsIoT`. `IsStaSupported` is `false` on Unix; on Windows it calls `CoInitializeEx(STA)`, balances it with `CoUninitialize` when it succeeds, and reports success unless the result is `E_NOTIMPL` (src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform@L16-L1124). +- **Directories.** On Windows, cache is `LocalApplicationData\Microsoft\PowerShell` and config is `Personal\PowerShell` (via `SafeDeriveFromSpecialFolder`, which returns an empty string when the base path is blank). On Unix, `SelectProductNameForDirectory(XDG_Type)` honours `XDG_CONFIG_HOME`, `XDG_DATA_HOME`, `XDG_CACHE_HOME` and otherwise uses `~/.config/powershell`, `~/.local/share/powershell` (+`Modules`), `~/.cache/powershell`; shared modules are the fixed path `/usr/local/share/powershell/Modules`; default-path directories are created on demand; an `UnauthorizedAccessException` returns the empty string, which makes the process working directory the fallback. `XDG_DATA_DIRS` is a documented TODO (issue 1060). If `HOME` is unset, a temporary home `pwsh--98288ff9-5712-4a14-9a11-23693b9cd91a` is created under the temp path (src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390). The home variable name differs: `HOME` on Unix, `USERPROFILE` on Windows (`CommonEnvVariableNames.Home`). +- **`Platform.GetFolderPath` on Unix** supports only `ProgramFiles` (returns `/bin` if present) and `MyDocuments` (home); any other special folder throws `NotSupportedException` (src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.GetFolderPath@L396-L408). +- **`NonWindows*` shim pattern.** `NonWindowsIsHardLink`, `NonWindowsGetUserFromPid`, `NonWindowsInternalGetLinkType`, `NonWindowsCreateSymbolicLink`, `NonWindowsCreateHardLink`, `NonWindowsSetDate`, `NonWindowsIsSameFileSystemItem`, `NonWindowsGetInodeData`, `NonWindowsIsExecutable`, `NonWindowsGetThreadId`, `NonWindowsGetProcessParentPid`, `NonWindowsKillProcess`, `NonWindowsWaitPid` keep Unix code out of Windows-origin source files. The file's own comment states the rule: use them only behind an `IsWindows` query. The parent pid comes from the native library on macOS and from `/proc//status` (`PPid:\t` line) on Linux; failure returns `-1`. +- **`CommonStat` model.** The Unix stat structure is copied into a managed class (inode, mode, uid, gid, link count, size, three times, block size, device, blocks, item type, setuid/setgid/sticky). `GetModeString()` returns an `ls`-style string with fast paths for three common modes (`-r--r--r--`, `-rw-r--r--`, `drwxr-xr-x`). Times that overflow fall back to the Unix epoch (a note names Raspbian). User and group names are resolved lazily through two **static, unsynchronized** `Dictionary` caches (src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.CommonStat@L587-L745). + +## 5. Data model + +No persistent model. Two global facts decide behavior: the compile-time symbol (fixed per assembly) and the cached runtime queries (`IsNanoServer`, `IsIoT`, `IsWindowsDesktop`, `IsStaSupported`; computed once per process). Per-process static caches on Unix: user-name map, group-name map, temp home string. + +## 6. Control flow + +```mermaid +flowchart TD + P[PowerShell.Common.props] -->|IsWindows true| W[Windows build: no UNIX symbol, Interop/Windows compiled] + P -->|else| U[Unix build: UNIX symbol, Windows-only files removed] + W --> R1[runtime: Platform.IsWindows true] + U --> R2[runtime: IsLinux or IsMacOS] + R1 --> D{headless SKU?} + D -->|Nano or IoT| H[no shell-execute fallback] + D -->|desktop| S[file association and ShellExecute fallback] +``` + +## 7. Algorithms + +Directory selection (§4). Native command executability: Unix asks the native library (`Platform.NonWindowsIsExecutable`); Windows compares the file extension with `PATHEXT` (default list `.exe .com .bat .cmd` when the variable is empty) (src/System.Management.Automation/engine/NativeCommandProcessor.cs::NativeCommandProcessor.IsExecutable@L1892-L1921). Command-name expansion adds each `PATHEXT` suffix only on Windows (`#if !UNIX` in `CommandSearcher`, around L1346). Everything else is selection by symbol; no algorithm of note. + +## 8. Observable behavior [SPEC] + +Behavior differences found in the code read (each `FACT-SRC`, run behavior `UNVERIFIED`): + +| area | Windows | Unix | +|---|---|---| +| native command fails to start | if the SKU is a desktop, look up the file association (`FindExecutable`), choose argument mode via `IsWindowsApplication`, allocate a hidden console when needed, then retry; headless SKU rethrows | rethrow the `Win32Exception` | +| non-zero exit text | exit code shown as `0x{0:X8}` | `0x{0:X2}` | +| screen scraping for minishell error stream | probed via the host raw UI | disabled (`false`) | +| child cleanup on dispose | background GUI apps keep running | the child is killed unless it runs in the background | +| `Invoke-Item` | `UseShellExecute` only on desktop SKUs | directories and non-executable files use `UseShellExecute` (xdg-open or open); a permission-denied (13) on start triggers that fallback | +| links | `WinCreateSymbolicLink`, `CreateHardLink`; failure code 1314 = elevation required | native library; elevation message when the error category is `PermissionDenied` | +| forbidden characters in remote-supplied item names | `\` `/` `:` | `\` `/` | +| mapped network drives | supported | `PlatformNotSupportedException` | +| `Set-ExecutionPolicy` store | registry or policy file | `PlatformNotSupportedException` | +| origin check (`IsLocalFile`) | security zone (MyComputer, Intranet, Trusted are local) | always local (no zone concept) | +| OEM encoding | code page from `GetOEMCP` | `Encoding.Default` | +| thread id | `GetCurrentThreadId` | native library call | +| break handler | console control handler (`ConsoleControl`) | `Console.CancelKeyPress` | +| `-File` target | run by path | a file without `.ps1` is read as text and run as a script (shebang support) | +| virtual terminal | n/a | DECCKM is switched on when the terminal supports VT | +| logging sink | ETW | syslog (`PSSysLogProvider`, `GetSysLogIdentity`, default keywords in config) | +| assembly load | GAC fallback (`TryFindInGAC`) | none | +| `STA` threads | supported if COM allows | not supported | + +Sources: src/System.Management.Automation/engine/NativeCommandProcessor.cs@L798-L830, @L1176-L1200, @L1408-L1430, @L1492-L1520, @L1855-L1921; src/System.Management.Automation/namespaces/FileSystemProvider.cs@L595-L606, @L1348-L1385, @L2393-L2423, @L5000-L5010; src/System.Management.Automation/security/SecurityManager.cs@L416-L433; src/System.Management.Automation/security/SecuritySupport.cs@L136-L145; src/System.Management.Automation/utils/PsUtils.cs@L103-L110; src/System.Management.Automation/utils/ClrFacade.cs@L116-L130; src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs@L1338-L1350, @L2028-L2042, @L2650-L2660. + +## 9. Error handling and edge cases + +- Directory creation failures on Unix are swallowed into an empty path: the process working directory then becomes the profile or cache location. This is silent (`INFERENCE`: a read-only home produces files in the current directory). +- `Win32Exception(Marshal.GetLastWin32Error())` is used on Unix too; the file comment states .NET maps `errno` to the message, so no manual mapping is needed. `Platform.Unix.GetErrorCategory(errno)` calls the native library to map `errno` to `ErrorCategory`. +- Stat time conversion catches any exception and substitutes the epoch. +- `IsHardLink` returns `false` for missing paths and directories and throws `Win32Exception` when the native call fails. + +## 10. Concurrency and threading + +Windows SKU flags use a nullable-bool cache without a lock (a benign race: same value). `IsStaSupported` and the cache directories are `Lazy`/static-initialized. The Unix user-name and group-name dictionaries are plain `Dictionary` with `Add` after `TryGetValue`: two threads formatting `Get-ChildItem` output at once can race (`INFERENCE`, not reproduced; `UNVERIFIED`, VB-072). + +## 11. Extension points and customization hooks + +None for end users. Contributors add Unix-only code behind `#if UNIX` and put the shim in `Platform`. Environment variables `XDG_*` and `HOME` steer Unix paths. `powershell.config.json` carries the syslog identity and keywords on Unix. + +## 12. Dependencies + +Inbound: nearly every subsystem that touches files, processes, consoles, links, logging or security zones. Outbound: .NET `OperatingSystem`, `Microsoft.Win32.Registry` (Windows), `libpsl-native` (Unix), `Interop.Windows` P/Invokes (Windows). + +## 13. Performance notes + +Mode-string fast paths and name caches exist to avoid P/Invoke per file when listing directories; the code comment says user and group lookups are done lazily "so we don't need to have a pinvoke for each file". + +## 14. Security considerations + +- Zone-based origin checks have **no Unix equivalent**: every file counts as local, so `RemoteSigned` semantics reduce to "no check" there (`INFERENCE` from `IsLocalFile`; the execution-policy read path on Unix is `UNVERIFIED`, VB-072). +- Execution policy cannot be set on Unix (`PlatformNotSupportedException`); policy there comes from other means (`UNVERIFIED`). +- Name rules for remote-supplied file names differ (`:` forbidden only on Windows because of alternate data streams), so a name safe on one OS is not safe on the other. +- Process kill on dispose (Unix) is a resource-safety measure; Windows leaves GUI children running. + +### Security surface + + +| Category | Seed hits | +|---|---:| +| execution-policy | 76 | +| signature | 39 | +| amsi | 25 | +| trust-boundary | 8 | +| app-control | 1 | +| **total** | 149 | + +Files with most hits: + +| File | Hits | +|---|---:| +| `src/System.Management.Automation/security/SecuritySupport.cs` | 94 | +| `src/System.Management.Automation/security/SecurityManager.cs` | 55 | + +These are regex scanner hits (`APPROX`, `verified: false`, no `SEC-ENF` ID yet). Phase 5 verifies them and assigns IDs; none is claimed here as an enforcement point. + + +What the new shell must guarantee here: security decisions must not silently weaken per OS. Where an OS lacks a concept (zones, registry policy), the shell must define an equivalent or state "not enforced" in one place and report it through the status API. + +## 15. Enterprise considerations + +Config and cache locations follow XDG on Unix and known folders on Windows, which matters for roaming profiles and containers (read-only home). Logging goes to ETW or syslog, not to one portable sink. Remoting differs by OS (WSMan and endpoint registration only on Windows; SSH on both). + +## 16. Tests + + +| Pester file | Describe | Context | It | +|---|---:|---:|---:| +| `test/powershell/Host/Base-Directory.Tests.ps1` | 2 | 2 | 2 | +| `test/powershell/Language/Scripting/NativeExecution/NativeCommandProcessor.Tests.ps1` | 6 | 0 | 15 | +| `test/powershell/Language/Scripting/NativeExecution/NativeLinuxCommands.Tests.ps1` | 2 | 0 | 8 | +| `test/powershell/Language/Scripting/NativeExecution/NativeUnixGlobbing.Tests.ps1` | 1 | 0 | 17 | +| `test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1` | 12 | 26 | 146 | +| `test/powershell/Modules/Microsoft.PowerShell.Security/ExecutionPolicy.Tests.ps1` | 5 | 8 | 21 | +| **total (6 files)** | 28 | 36 | 209 | + +| xUnit file | Facts | Theories | Classes | +|---|---:|---:|---| +| `test/xUnit/csharp/test_CorePsPlatform.cs` | 12 | 0 | `PlatformTests` | +| `test/xUnit/csharp/test_FileSystemProvider.cs` | 9 | 0 | `FileSystemProviderTests` | +| `test/xUnit/csharp/test_PSConfiguration.cs` | 15 | 0 | `PowerShellPolicyFixture`, `PowerShellPolicyTests` | + +Matching is by path keyword (`APPROX`): `FileSystem.Tests`, `NativeCommandProcessor`, `NativeLinuxCommands`, `NativeUnixGlobbing`, `Base-Directory`, `ExecutionPolicy.Tests`, `test_CorePsPlatform`, `test_FileSystemProvider`, `test_PSConfiguration`. + + +## 17. Design decisions and trade-offs + +- `FACT-SRC`: build-time split removes dead code and avoids shipping Windows P/Invokes on Unix, at the price of **one managed assembly per OS family** and 390 conditional blocks that are only compiled on one OS (so a Windows-only developer cannot compile-check the Unix branches). +- `FACT-SRC`: runtime queries (`OperatingSystem.*`) are used where both branches must compile on both OSes (for example `GetNativeDllSubFolderName`, `NonWindowsGetProcessParentPid`). +- `FACT-SRC`: the `NonWindows*` shims keep the original Windows files readable and move Unix code to one class (the file comment gives this as the reason). +- `INFERENCE`: because some Windows-only features (COM, WMI adapter, session-configuration registration) are removed by file, the public surface differs per OS without a single capability table. + +## 18. Guidance for the new shell [LESSON] + +- **LSN-PLAT-01** [REDESIGN] (area: NLD-PLAT) Isolate OS differences behind one platform interface (paths, process launch, links, console, logging, identity) with a runtime capability table, and keep conditional compilation for the interface implementations only. Evidence: src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform@L16-L1124. DEV: DEV-PLAT-230. +- **LSN-PLAT-02** [REPLICATE] (area: NLD-PLAT) Honour XDG directories on Unix and known folders on Windows, create on demand, and report (not hide) a failure to create them. Evidence: src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390. +- **LSN-PLAT-03** [REDESIGN] (area: NLD-SEC) Define the security-zone and execution-policy concept for every OS the shell supports, or declare it unsupported through one status API. DEV: DEV-PLAT-233. Evidence: src/System.Management.Automation/security/SecurityManager.cs@L416-L433. +- **LSN-PLAT-04** [ADAPT] (area: NLD-PLAT) Replace unsynchronized static caches for lazily resolved OS data with a concurrent map or a per-call lookup. Evidence: src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.CommonStat@L587-L745. + +## 19. Open questions / UNVERIFIED items + +- `UNVERIFIED` (VB-072): the execution-policy read path on Unix, other `UNIX` branches in `SecuritySupport.cs` (lines 282–428, 1344–1655), `Utils.cs`, `ProviderBase.cs`, `CommandDiscovery.cs`; Unix-only `FileSystemProvider` blocks other than those listed in §8; the rest of the 390 conditional blocks; whether the user-name cache race is reachable. +- `UNVERIFIED` (VB-073): Windows-only behavior of `Process.cs`, `Service.cs`, `Computer.cs` (`ComputerUnix.cs` exists for Unix); how the build selects the `IsWindows` property in CI and packaging (`build.psm1` was not read for this). + +Verification: `CorePsPlatform.cs` lines 1–1125 were read in full; `PowerShell.Common.props` lines 160–175; the csproj lines 20–90; `NativeCommandProcessor.cs`, `FileSystemProvider.cs`, `SecurityManager.cs`, `SecuritySupport.cs` (lines 132–145 only), `PSConfiguration.cs` (250–262, 375–383), `PsUtils.cs` (103–110), `ClrFacade.cs` (116–130), `CommandSearcher.cs` (1340–1358), `ConsoleHost.cs` (1338–1350, 2028–2042, 2650–2660), `CorePsAssemblyLoadContext.cs` (98–108, 205–260, 300–370, 539–566) were read in the windows quoted. Counts are from `grep` and `data/conditional-compilation.json`. Test counts from `data/tests.json` through `tools/gen_T3.py`. diff --git a/docs-out/09-synthesis/deviation-register.md b/docs-out/09-synthesis/deviation-register.md index f42aa11..4c13e18 100644 --- a/docs-out/09-synthesis/deviation-register.md +++ b/docs-out/09-synthesis/deviation-register.md @@ -46,6 +46,9 @@ All rows are `PROPOSED` (the only status this run may write). `APPROVED`/`REJECT | DEV-GRM-010 | PROPOSED | The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing. | Parsing is pure: directives are recorded in the AST and resolved in a later phase. | Deterministic, side-effect-free parse; safe for tooling and untrusted text. | Errors for a missing assembly appear at a later phase than in PowerShell. | CF-GRM-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PARSER-07 (02-language/02.2-parser.md); LSN-QRK-03 (02-language/SEMANTIC-QUIRKS.md) | | DEV-HELP-165 | PROPOSED | Updatable help downloads unsigned packages from a module-provided URI. | Signed help packages with a catalog and offline mode, verified before install. | Integrity of installed help. | Help authors must sign packages. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTHELP-04 (03-runtime/help-system.md) | | DEV-HELP-166 | PROPOSED | Help is authored in MAML XML plus a DSC resource provider. | Lighter structured format and comment-based help only. | Simpler authoring and parsing. | Existing MAML help needs conversion. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTHELP-05 (03-runtime/help-system.md) | +| DEV-HOST-200 | PROPOSED | ConsoleHost parses switches that widen access (for example -noprofile, -executionpolicy, -configurationfile, server modes) from the command line, then starts; policy applies after parsing. | Load policy from a signed source first; refuse switches that widen access when lockdown is on; refuse server modes without an authenticated transport. | Closes a path where a launch argument weakens a locked-down start. | Scripts that pass widening switches under lockdown stop working. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-HOSTCON-05 (04-hosting/console-host.md) | +| DEV-HOST-201 | PROPOSED | ConsoleHost carries Windows-only pieces: ConsoleControl Win32 calls, jump list, -windowstyle, -sta by default, Hyper-V socket modes. | Leave them out of the first version. | Smaller host, fewer native calls. | No Windows console extras, no Hyper-V mode. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-HOSTCON-06 (04-hosting/console-host.md) | +| DEV-JEA-220 | PROPOSED | Role merging concatenates values from all matching roles; there is no deny rule. | Add explicit deny rules with deny-over-allow precedence and a report of effective capabilities. | Predictable least privilege. | Role files need a deny section; merge results change when deny is used. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-JEA-03 (05-remoting/jea.md) | | DEV-JOB-190 | PROPOSED | Background jobs start a separate pwsh process and parallel tasks use runspaces created from the session; policy inheritance is not enforced in the code read. | Child processes and parallel runspaces inherit policy, language mode and visibility as a hard rule, with per-job quotas. | Prevents escape from a restricted session through jobs. | Jobs cannot be more privileged than the parent. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTJOB-04 (03-runtime/jobs-and-concurrency.md) | | DEV-JOB-191 | PROPOSED | ThrottlingJob, ContainerParentJob and a job source-adapter registry exist. | One local job type plus the parallel pool. | Smaller job model. | Third-party job sources unsupported initially. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTJOB-05 (03-runtime/jobs-and-concurrency.md) | | DEV-LEX-001 | PROPOSED | Tokenizer detects signature-block lookalikes and processes #requires comment lines at parse time. | Omit both from the tokenizer; if wanted, handle pragmas in a dedicated post-tokenize pass. | Keeps the tokenizer a pure text-to-token function; removes security-sensitive fuzzy matching from the hot path. | Scripts using #requires get no version/module check; signed-script trailer detection unavailable. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOK-07 (02-language/02.1-tokenizer.md) | @@ -53,12 +56,20 @@ All rows are `PROPOSED` (the only status this run may write). `APPROVED`/`REJECT | DEV-MOD-171 | PROPOSED | Windows PowerShell compatibility imports, .cdxml CIM modules and DSC resource exports are supported. | Omit in the first version. | Smaller scope; removes Windows-only machinery. | Those modules cannot be imported. | CF-MOD-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTMOD-07 (03-runtime/modules.md) | | DEV-PIPE-110 | PROPOSED | Host-facing object streams are effectively unbounded (capacity Int32.MaxValue) and a pipeline has no time or memory budget. | Bounded, cancellable host streams by default plus an optional per-pipeline time/memory budget. | Predictable resource use in enterprise hosts. | Behavior differs for producers that exceed the bound; hosts must handle blocking or rejection. | CF-LCY-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPIPE-06 (03-runtime/runspace-and-pipeline.md) | | DEV-PIPE-111 | PROPOSED | An internal empty "pulse" pipeline is created to run pending engine events when the runspace is idle. | Schedule event processing on the runspace loop without creating a pipeline. | Removes a special pipeline and its state-machine corner cases. | Code that observed the pulse pipeline (id 0, history) would not see it. | CF-LCY-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPIPE-08 (03-runtime/runspace-and-pipeline.md) | +| DEV-PLAT-230 | PROPOSED | OS differences are spread over 390 conditional blocks and many runtime queries. | One platform interface with a runtime capability table; conditional compilation only inside interface implementations. | Testable on every OS; one place to review. | Larger initial design effort. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PLAT-01 (06-platform/windows-unix-split.md) | +| DEV-PLAT-231 | PROPOSED | Native layer: hand-kept struct layouts for libpsl-native, three interop styles, C source in another repository. | One native surface per OS with generated bindings and a layout test in CI. | Fewer ABI mismatches. | Build tooling must generate and verify bindings. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-NAT-01 (06-platform/native-interop.md) | +| DEV-PLAT-232 | PROPOSED | Last-resort resolver loads /. from the folder of the calling assembly with no signature or location check. | Prefer the application directory and a signed or pinned per-RID folder; refuse writable locations unless policy allows. | Blocks library planting through writable module folders. | Modules with native libraries need a signed or allowed location. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-NAT-04 (06-platform/native-interop.md) | +| DEV-PLAT-233 | PROPOSED | Zone checks and the execution-policy store exist only on Windows; Unix treats every file as local and cannot set a policy. | Define origin and policy for every supported OS, or report 'not enforced' through one status API. | Security behavior is the same or explicitly different per OS. | Unix users get a policy store and origin marks. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PLAT-03 (06-platform/windows-unix-split.md) | | DEV-PROV-140 | PROPOSED | Providers receive a mutable Context set by the engine and have no policy mediation layer. | A mandatory policy mediation layer between engine and providers and an explicit per-call context object. | Root confinement, rights and audit enforced outside provider code. | Provider authors must target a new context type. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPROV-03 (03-runtime/provider-model.md) | | DEV-PROV-141 | PROPOSED | File-system drives are auto-mounted on lookup miss. | Mount drives explicitly or from configuration. | Predictable drive table; no hidden system probing. | Scripts using unmounted drive letters need explicit mounts. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTPROV-06 (03-runtime/provider-model.md) | +| DEV-REM-210 | PROPOSED | PSRP carries XML payloads, fragments, in-protocol key exchange and priority streams over several transports. | No remoting in the first version; if added later, use a typed binary protocol with transport-level authentication. | Removes a large attack surface and a large unverified area. | No remote sessions initially. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PSRP-01 (05-remoting/psrp-protocol.md) | +| DEV-REM-211 | PROPOSED | Transports: WSMan (native plugin), SSH subsystem, named pipe, Hyper-V socket, container and VM process. | Omit WSMan, Hyper-V and container transports; consider SSH-style process stdio first if remoting is added. | Avoids Windows-only native components. | No WSMan or Hyper-V endpoints. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TRANS-01 (05-remoting/transports.md) | | DEV-RTOPS-051 | PROPOSED | Classes are emitted with Reflection.Emit and method bodies wrapped by ScriptBlockMemberMethodWrapper. | Classes are interpreter objects, not CLR types. | No Reflection.Emit dependency. | Classes are not usable from other .NET code. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTOPS-05 (02-language/RUNTIME-OPERATIONS.md) | | DEV-SB-090 | PROPOSED | ScriptBlock can become a PowerShell object and a steppable pipeline. | Omitted initially. | Less surface. | No GetPowerShell/GetSteppablePipeline. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SB-05 (02-language/SCRIPT-BLOCKS.md) | | DEV-SCP-080 | PROPOSED | Values assigned to global variables are marked untrusted in Constrained Language Mode. | No trust marking. | Simpler assignment path. | No Constrained Language Mode. | CF-SCP-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SCOPE-07 (02-language/SCOPE-AND-VARIABLES.md) | +| DEV-SDK-205 | PROPOSED | The default embedded session is full language with all commands visible. | Default embedded session is restricted; full language needs an explicit opt-in. | Safe default for host applications. | Embedders must opt in to the full session. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-HOSTSDK-03 (04-hosting/sdk.md) | | DEV-SEM-030 | PROPOSED | `using module` is resolved at parse time by running Get-Module -ListAvailable in a PowerShell instance. | The directive is recorded; module import happens in a later phase. | Parsing has no side effects or I/O. | Missing-module errors appear later than in PowerShell. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SEM-06 (02-language/02.4-semantic-analysis.md) | +| DEV-SER-215 | PROPOSED | CLIXML reader has depth limits but no explicit maximum input size; depth-1 property bags can include sensitive properties. | Add a maximum input size and streaming limit to the reader; mark sensitive properties so shallow bags cannot leak them. | Bounds memory and leakage. | Large documents fail to import unless a limit is raised. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CLIXML-04 (05-remoting/serialization-clixml.md) | | DEV-TIER-060 | PROPOSED | Hot loops are promoted from interpreter to JIT in the background mid-execution via in-place instruction replacement. | No mid-execution promotion; any optimizer runs synchronously at defined points. | Deterministic execution, simpler frames. | Long top-level loops lose the speed-up PowerShell gets. | CF-CMP-002 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TIER-04 (02-language/TIERING-POLICY.md) | | DEV-TOOLS-100 | PROPOSED | PSParser.Tokenize returns PowerShell 2 style tokens by rewriting the saved token list. | Only the modern token list. | No V2 branches in the parser. | Legacy tools unsupported. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOOLS-03 (02-language/02.8-language-tooling.md) | | DEV-TYP-070 | PROPOSED | Constrained Language Mode restricts conversion targets inside FigureConversion. | No language mode in the conversion layer; allow-list at the type-system boundary. | Simpler conversion code. | No Constrained Language Mode. | CF-CNV-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CONV-05 (02-language/CONVERSION-RULES.md) | diff --git a/docs-out/DECISIONS.md b/docs-out/DECISIONS.md index 5f9c386..b013c8d 100644 --- a/docs-out/DECISIONS.md +++ b/docs-out/DECISIONS.md @@ -26,3 +26,6 @@ | D-022 | `docs-out/tools/gen_T3.py` with `docs-out/tools/annotations/T3-docs.json` renders the Pester/xUnit test tables (``) and the per-subsystem security-seed tables (``) of all 03-runtime documents from `data/tests.json` and `data/enforcement-points.json`. Seed hits are scanner output, unverified, and carry no `SEC-ENF` ID; Phase 5 assigns IDs. No `SEC-ENF-*` ID is invented in Phase 3. | Rule 9 (no hand-typed catalogs) and rule 1 (evidence only). | | D-023 | CF-LCY-001 is closed (statement fully read); CF-DSC-001, CF-BND-001, CF-ETS-001, CF-MOD-001, CF-ERR-001 stay open because parts of their algorithms are listed in VB-027, VB-030, VB-038, VB-046, VB-049. CF-ERR-001 references the statement-level protocol documented in RUNTIME-OPERATIONS §6.1 instead of restating it. | CF closure requires a fully read statement (brief rule 5). | | D-024 | The study prompt lists `Workflow` among command types; `CommandTypes` in v7.6.9 has no such member (CON-014). The native process start point (prepare phase) and the 256-entry `$Error` cap are recorded as claims (`T3.jsonl`). | Source wins over prompt hints. | +| D-025 | Phase 4 documents (`04-hosting/`, `05-remoting/`, `06-platform/`) are authored inline at tier B depth. Remoting documents carry `OUT-OF-SCOPE-DOCUMENTED` except CLIXML (`EXTENDED`); platform documents are `EXTENDED`. `10-command-modules/` was generated in Phase 0 and is not rewritten. | Remoting is not in the first version; the mechanisms (restricted sessions, serialization) still inform the design. | +| D-026 | `gen_T3.py` keys T41..T49 cover the Phase 4 documents; T41..T43 hosting, T44..T47 remoting, T48..T49 platform. Security-seed tables are scanner output without `SEC-ENF` IDs. | Same rule as D-022. | +| D-027 | Lesson rule: ADAPT carries no DEV; AVOID and REDESIGN carry one. Phase 4 DEV ids: DEV-HOST-200/201, DEV-SDK-205, DEV-REM-210/211, DEV-SER-215, DEV-JEA-220, DEV-PLAT-230..233. | Keeps the coordinator lesson check at 0 problems. | diff --git a/docs-out/PROGRESS.md b/docs-out/PROGRESS.md index c0a76d4..155a3b6 100644 --- a/docs-out/PROGRESS.md +++ b/docs-out/PROGRESS.md @@ -46,7 +46,16 @@ T21 tokenizer | T22 parser/grammar/errors | T23 AST | T24 semantic | T25 compile | T37 | `03-runtime/modules.md` (CF-MOD-001) | draft (tier B); DEV-MOD-170/171; VB-045..047 | | T38 | `03-runtime/errors-and-diagnostics.md` (CF-ERR-001) | draft (tier B); DEV-ERR-180; VB-048..050 | | T39 | `03-runtime/jobs-and-concurrency.md`, `03-runtime/debugger.md`, `03-runtime/eventing.md` | draft (tier B); DEV-JOB-190/191, DEV-DBG-195/196, DEV-EVT-198/199; VB-051..057 | - -Generated parts of 03-runtime (test tables, security-seed tables): `docs-out/tools/gen_T3.py` + `docs-out/tools/annotations/T3-docs.json` (D-022). - -Next step: Phase 4 (hosting, SDK, remoting, platform, command-module families), Phase 5 security (assign `SEC-ENF` IDs to verified seed hits), and the deeper reads in VB-017..VB-023 and VB-024..VB-057. Open: golden samples and oracle remain waived (VB-001). +| T41 | `04-hosting/host-abstraction.md` (HOST-ABSTRACTION-001) | draft (tier B); DEV-HOST-200 shared with T42; VB-058/059 | +| T42 | `04-hosting/console-host.md` (HOST-CONSOLE-001) | draft (tier B); DEV-HOST-200/201; VB-060/061 | +| T43 | `04-hosting/sdk.md` (HOST-SDK-001) | draft (tier B); DEV-SDK-205; VB-062/063 | +| T44 | `05-remoting/psrp-protocol.md` (REM-PSRP-001, out of scope, documented) | draft (tier B); DEV-REM-210; VB-064/065 | +| T45 | `05-remoting/transports.md` (REM-TRANSPORT-001, out of scope, documented) | draft (tier B); DEV-REM-211; VB-066/067 | +| T46 | `05-remoting/serialization-clixml.md` (REM-CLIXML-001, extended) | draft (tier B); DEV-SER-215; VB-068/069 | +| T47 | `05-remoting/jea.md` (REM-JEA-001, out of scope, documented) | draft (tier B); DEV-JEA-220; VB-070/071 | +| T48 | `06-platform/windows-unix-split.md` (PLAT-SPLIT-001) | draft (tier B); DEV-PLAT-230, DEV-PLAT-233; VB-072/073 | +| T49 | `06-platform/native-interop.md` (PLAT-NATIVE-001) | draft (tier B); DEV-PLAT-231, DEV-PLAT-232; VB-074/075 | + +Generated parts of 03-runtime and Phase 4 documents (test tables, security-seed tables): `docs-out/tools/gen_T3.py` + `docs-out/tools/annotations/T3-docs.json` (D-022, D-026). + +Phase 4 status: hosting, remoting and platform documents (T41..T49) written; command-module families were generated in Phase 0 (D-025). Next step: Phase 5 security (assign `SEC-ENF` IDs to verified seed hits), and the deeper reads in VB-017..VB-023 and VB-024..VB-075. Open: golden samples and oracle remain waived (VB-001). diff --git a/docs-out/README.md b/docs-out/README.md index 7760c28..92a3ed2 100644 --- a/docs-out/README.md +++ b/docs-out/README.md @@ -17,6 +17,12 @@ Pin: version `v7.6.9`, commit `523031e` (`.work/pin.json`). STATIC-ONLY run: not 2. State and types: [execution-context-and-session-state.md](03-runtime/execution-context-and-session-state.md), [extended-type-system.md](03-runtime/extended-type-system.md) (CF-ETS-001), [modules.md](03-runtime/modules.md) (CF-MOD-001), [provider-model.md](03-runtime/provider-model.md) 3. Output and tooling: [formatting-and-output.md](03-runtime/formatting-and-output.md), [help-system.md](03-runtime/help-system.md), [jobs-and-concurrency.md](03-runtime/jobs-and-concurrency.md), [debugger.md](03-runtime/debugger.md), [eventing.md](03-runtime/eventing.md) +## Hosting, remoting and platform (`04-hosting/`, `05-remoting/`, `06-platform/`) + +1. Hosting: [host-abstraction.md](04-hosting/host-abstraction.md) → [console-host.md](04-hosting/console-host.md) → [sdk.md](04-hosting/sdk.md) +2. Remoting (out of first-version scope, documented): [psrp-protocol.md](05-remoting/psrp-protocol.md) → [transports.md](05-remoting/transports.md) → [serialization-clixml.md](05-remoting/serialization-clixml.md) → [jea.md](05-remoting/jea.md) +3. Platform: [windows-unix-split.md](06-platform/windows-unix-split.md) → [native-interop.md](06-platform/native-interop.md) + ## Repository overview (`00-overview/`, `01-repository/`, `10-command-modules/`) [repo-census](00-overview/repo-census.md), [source-tree-map](00-overview/source-tree-map.md), [project-dependency-graph](00-overview/project-dependency-graph.md), [external-dependencies](00-overview/external-dependencies.md), [layered-architecture](00-overview/layered-architecture.md), [build-system](01-repository/build-system.md), [ci-cd-and-packaging](01-repository/ci-cd-and-packaging.md), [coding-standards](01-repository/coding-standards.md), [command module families](10-command-modules/_index.md). diff --git a/docs-out/VERIFICATION-BACKLOG.md b/docs-out/VERIFICATION-BACKLOG.md index 998056b..c4e1139 100644 --- a/docs-out/VERIFICATION-BACKLOG.md +++ b/docs-out/VERIFICATION-BACKLOG.md @@ -61,3 +61,21 @@ Every `UNVERIFIED` claim and every `BLOCKED` item. Under STATIC-ONLY (owner deci | VB-055 | OPEN | RT-DEBUGGER-001 §19 | Breakpoint cmdlets, `Debug-Runspace`, `Wait-Debugger`, host prompt integration. | Read the utility cmdlets and ConsoleHost debugger code. | Debugger front end. | | VB-056 | OPEN | RT-EVENTING-001 §19 | `OnIdle` timer logic, `UnsubscribeEvent`, `GenerateEventHandler` (IL emission), `PSEventSubscriber` members, `PSEventJob`, `PSRemoteEventManager`, `Register-ObjectEvent` parameters, `PSEventArgsCollection`. | Read `EventManager.cs` remaining ranges. | Eventing detail. | | VB-057 | OPEN | RT-EVENTING-001 §19 | Where the `Exiting` engine event is raised and who consumes `OnScriptBlockInvoke`. | Search for `PSEngineEvent.Exiting` and `OnScriptBlockInvoke` uses. | Engine event consumers. | +| VB-058 | OPEN | HOST-ABSTRACTION-001 §19 | `InternalHost` exit-nested-prompt path and remaining members, `InternalHostUserInterface` buffering, `PSHostUserInterface` transcription beyond the cited hooks, `HostUtilities`, `DefaultHost`. | Read `InternalHost.cs`, `InternalHostUserInterface.cs`, `HostUtilities.cs`, `DefaultHost`. | Host contract details. | +| VB-059 | OPEN | HOST-ABSTRACTION-001 §19 | `InitialSessionState` entry classes, `Bind`, `CreateFromSessionConfigurationFile` body, snap-in handling, `Clone` semantics. | Read `InitialSessionState.cs` remaining ranges. | Session construction. | +| VB-060 | OPEN | HOST-CONSOLE-001 §19 | `ConsoleHostUserInterface`, `ConsoleHostRawUserInterface`, `ConsoleControl`, progress rendering, `CommandLineParameterParser` value handling, `Executor` async details, `PushRunspace`/`PopRunspace`, `OnExecutionSuspended`, transcript, native launcher. | Read the ConsoleHost project files not listed in the reading ledger. | Console host behavior. | +| VB-061 | OPEN | HOST-CONSOLE-001 §19 | PSReadLine (outside this repository) and prompt/feedback evaluation. | Read the PSReadLine source or its package docs. | Interactive editing. | +| VB-062 | OPEN | HOST-SDK-001 §19 | `PowerShell` reuse after completion, `PSCommand` semantics, `PSDataCollection` internals, `PSDataStreams`, `HadErrors`, `RunspacePool` API details, `Connect`/disconnect, `Runspace` events. | Read `PowerShell.cs`, `PSCommand`, `PSDataCollection`, `RunspacePool` sources. | Embedding API contract. | +| VB-063 | OPEN | HOST-SDK-001 §19 | `SessionStateProxy` concurrency checks, the sample app, per-RID publishing, reference assemblies. | Read the SDK project, sample app and packaging scripts. | SDK packaging. | +| VB-064 | OPEN | REM-PSRP-001 §19 | Session state-machine tables (client and server), key exchange conditions, runspace-pool and pipeline driver message handling, client classes, `RemoteHost` encoding, `EncodeAndDecode` builders, `ThrottleManager`. | Read `ServerRunspacePoolDriver`, `ServerPowerShellDriver`, client session and `EncodeAndDecode.cs` bodies. | Remoting protocol detail. | +| VB-065 | OPEN | REM-PSRP-001 §19 | `Defragmentor` reassembly error paths, data size limits and timeouts, `PSSenderInfo` construction. | Read `fragmentor.cs` 280-965 and `PSPrincipal.cs`. | Resource limits. | +| VB-066 | OPEN | REM-TRANSPORT-001 §19 | `WSManTransportManager`, `WSManPlugin*`, `WSManNativeAPI`, Hyper-V socket classes, `ContainerProcess`, VM and container details, `PSRemotingCmdlet`, `InvokeCommandCommand`, `newrunspacecommand`, session option defaults. | Read the named files. | Transports. | +| VB-067 | OPEN | REM-TRANSPORT-001 §19 | Exact `ssh` argument order in `StartSSHProcess`, `OutOfProcessMediatorBase` read loop, `NewProcessConnectionInfo`. | Read `RunspaceConnectionInfo.cs` and `OutOfProcServerMediator.cs` remaining ranges. | SSH transport. | +| VB-068 | OPEN | REM-CLIXML-001 §19 | `HandleComplexTypePSObject`, `HandleKnownContainerTypes`, enum handling, CIM instance serialization, `ReadListContainer`/`ReadDictionary` result types, `ReadProperties`, `TNRef` table, `Deserialized.` helpers, `PSPrimitiveDictionary` restrictions, SecureString protection on Unix, script block default path. | Read `serialization.cs` remaining ranges. | Type fidelity. | +| VB-069 | OPEN | REM-CLIXML-001 §19 | `Export-Clixml`, `Import-Clixml`, `PSSerializer` corner cases. | Read the cmdlet sources and tests. | Serialization entry points. | +| VB-070 | OPEN | REM-JEA-001 §19 | How `CommandModifications` apply at run time, `VisibleExternalCommands` mapping, `FunctionDefinitions` proxies, virtual and group-managed accounts, `RequiredGroups` SDDL, `ValidateRoleDefinitions`, `LoadConfigFile` evaluation, rest of `GetInitialSessionState` (2200-2450). | Read the named ranges and registration cmdlets. | Restricted sessions. | +| VB-071 | OPEN | REM-JEA-001 §19 | `PSPrincipal`/`PSSenderInfo` construction, `New-PSRoleCapabilityFile`, `Test-PSSessionConfigurationFile`. | Read `PSPrincipal.cs` and the cmdlets. | Identity in restricted sessions. | +| VB-072 | OPEN | PLAT-SPLIT-001 §19 | Execution-policy read path on Unix, other `UNIX` branches in `SecuritySupport.cs`, `Utils.cs`, `ProviderBase.cs`, `CommandDiscovery.cs`, Unix-only `FileSystemProvider` blocks beyond those listed, the rest of the 390 conditional blocks, reachability of the user-name cache race. | Read each `#if UNIX` block listed in `data/conditional-compilation.json`. | Per-OS behavior. | +| VB-073 | OPEN | PLAT-SPLIT-001 §19 | Windows-only behavior of `Process.cs`, `Service.cs`, `Computer.cs`; how CI and packaging set `IsWindows`. | Read those files and `build.psm1`. | Per-OS features. | +| VB-074 | OPEN | PLAT-NATIVE-001 §19 | C source and exported symbols of `libpsl-native` (not in this tree), contents of `Microsoft.PowerShell.Native`, default probing of `libpsl-native`, Windows native components. | Read the PowerShell-native repository or unpack the package. | Native layer. | +| VB-075 | OPEN | PLAT-NATIVE-001 §19 | Bodies of 30 of 33 `Interop/Windows` files, `security/nativeMethods.cs`, `WinTrust.cs`, `ConsoleControl.cs`, `WSManNativeAPI.cs`, `PdhHelper.cs`, `UnsafeNativeMethods.cs`. | Read the named files. | Windows interop. | diff --git a/docs-out/data/claims/T4.jsonl b/docs-out/data/claims/T4.jsonl new file mode 100644 index 0000000..19630f1 --- /dev/null +++ b/docs-out/data/claims/T4.jsonl @@ -0,0 +1,12 @@ +{"id": "CLM-PLT-0001", "subject_key": "platform.cache_dir_unix_default", "predicate": "default cache directory on Unix when XDG_CACHE_HOME is unset", "value": "~/.cache/powershell", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0002", "subject_key": "platform.shared_modules_unix", "predicate": "shared modules path on Unix", "value": "/usr/local/share/powershell/Modules", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0003", "subject_key": "platform.getfolderpath_unix_supported", "predicate": "special folders supported by Platform.GetFolderPath on Unix", "value": "ProgramFiles, MyDocuments", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.GetFolderPath@L396-L408"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0004", "subject_key": "platform.native_lib_name_unix", "predicate": "native library name imported by Platform.Unix.NativeMethods", "value": "libpsl-native", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0005", "subject_key": "platform.native_libraryimport_count", "predicate": "LibraryImport entries in Platform.Unix.NativeMethods", "value": "17", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0006", "subject_key": "platform.native_rid_folders", "predicate": "native resolver folder prefixes", "value": "win-, linux-, osx- plus lower-case process architecture", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext@L19-L565"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0007", "subject_key": "platform.pinvoke_dll_name_constants", "predicate": "string constants in PinvokeDllNames", "value": "125", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/utils/PInvokeDllNames.cs::PinvokeDllNames@L13-L140"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0008", "subject_key": "platform.interop_windows_files", "predicate": "source files under engine/Interop/Windows", "value": "33", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Interop/Windows/CreateHardLink.cs"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0009", "subject_key": "platform.native_package", "predicate": "NuGet package that supplies native components", "value": "Microsoft.PowerShell.Native 700.0.2", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/System.Management.Automation.csproj@L44-L46"], "doc": "PLAT-NATIVE-001", "cf": null} +{"id": "CLM-PLT-0010", "subject_key": "platform.unix_nonzero_exit_hex_width", "predicate": "hex width of the exit code text on Unix", "value": "2 digits (0x{0:X2}); Windows uses 8 digits", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/NativeCommandProcessor.cs@L1176-L1200"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0011", "subject_key": "platform.unix_origin_check", "predicate": "IsLocalFile result on Unix", "value": "always true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/security/SecurityManager.cs@L416-L433"], "doc": "PLAT-SPLIT-001", "cf": null} +{"id": "CLM-PLT-0012", "subject_key": "platform.unix_set_execution_policy", "predicate": "SetExecutionPolicy on Unix", "value": "throws PlatformNotSupportedException", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/security/SecuritySupport.cs@L136-L145"], "doc": "PLAT-SPLIT-001", "cf": null} diff --git a/docs-out/data/lessons.json b/docs-out/data/lessons.json index db7e981..dc97e2a 100644 --- a/docs-out/data/lessons.json +++ b/docs-out/data/lessons.json @@ -1099,6 +1099,204 @@ "file": "03-runtime/runspace-and-pipeline.md", "line": 260 }, + { + "id": "LSN-JEA-01", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Deny-by-default restricted sessions built from a declarative data file: empty baseline, named visible commands, parameter constraints, language mode, execution policy, startup scripts and transcripts.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.GetInitialSessionState@L1979-L2451.", + "dev": null, + "doc": "REM-JEA-001", + "file": "05-remoting/jea.md", + "line": 161 + }, + { + "id": "LSN-JEA-02", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Visibility as a per-command property (public/private) enforced at resolution and invocation, so internal helpers stay callable.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.ProcessVisibleCommand@L2614-L2650.", + "dev": null, + "doc": "REM-JEA-001", + "file": "05-remoting/jea.md", + "line": 162 + }, + { + "id": "LSN-JEA-03", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Add explicit deny rules and a defined precedence (deny over allow) to role merging, plus a \"what can this role do\" report. DEV: DEV-JEA-220.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.MergeConfigHashIntoConfigHash@L1887-L1929.", + "dev": "DEV-JEA-220", + "doc": "REM-JEA-001", + "file": "05-remoting/jea.md", + "line": 163 + }, + { + "id": "LSN-JEA-04", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "Make parameter constraints part of the command contract (typed policy objects) and enforce them in the binder, instead of attribute-name keys in a hashtable.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCPowerShellConfiguration.ProcessCommandModification@L2527-L2612.", + "dev": null, + "doc": "REM-JEA-001", + "file": "05-remoting/jea.md", + "line": 164 + }, + { + "id": "LSN-JEA-05", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "Sign configuration and role files and verify them at load.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs::DISCUtils.LoadConfigFile@L1453-L1474.", + "dev": null, + "doc": "REM-JEA-001", + "file": "05-remoting/jea.md", + "line": 165 + }, + { + "id": "LSN-PSRP-01", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Do not build remoting in the first version; if built later, do not copy PSRP's XML payload design or its in-protocol key exchange. DEV: DEV-REM-210.", + "evidence": "src/System.Management.Automation/engine/remoting/common/WireDataFormat/EncodeAndDecode.cs::RemotingConstants@L75-L113.", + "dev": "DEV-REM-210", + "doc": "REM-PSRP-001", + "file": "05-remoting/psrp-protocol.md", + "line": 190 + }, + { + "id": "LSN-PSRP-02", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "Keep the lesson that protocol versions are explicit and negotiated downward with a published history, and that security belongs to a mutually authenticated encrypted transport.", + "evidence": "src/System.Management.Automation/engine/remoting/server/serverremotesession.cs::ServerRemoteSession.RunServerNegotiationAlgorithm@L967-L1058.", + "dev": null, + "doc": "REM-PSRP-001", + "file": "05-remoting/psrp-protocol.md", + "line": 191 + }, + { + "id": "LSN-PSRP-03", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Keep length-prefixed fragmentation with start/end flags and a priority lane for control and prompt traffic.", + "evidence": "src/System.Management.Automation/engine/remoting/common/fragmentor.cs::FragmentedRemoteObject@L29-L426.", + "dev": null, + "doc": "REM-PSRP-001", + "file": "05-remoting/psrp-protocol.md", + "line": 192 + }, + { + "id": "LSN-PSRP-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Forward host calls (prompts, progress) as typed RPCs with call ids if a remote host UI is ever needed.", + "evidence": "src/System.Management.Automation/engine/remoting/common/WireDataFormat/RemoteHost.cs@L1-L762.", + "dev": null, + "doc": "REM-PSRP-001", + "file": "05-remoting/psrp-protocol.md", + "line": 193 + }, + { + "id": "LSN-CLIXML-01", + "verdict": "ADAPT", + "area": "NLD-TYP", + "text": "Keep the idea of a self-describing, depth-limited, reference-preserving interchange format with exact primitives and inert property bags for the rest, but choose a schema'd format (for example JSON or CBOR with a type vocabulary) and keep CLIXML import/export as a compatibility adapter.", + "evidence": "src/System.Management.Automation/engine/serialization.cs::InternalSerializer.WriteOneObject@L971-L1042.", + "dev": null, + "doc": "REM-CLIXML-001", + "file": "05-remoting/serialization-clixml.md", + "line": 164 + }, + { + "id": "LSN-CLIXML-02", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Never instantiate types from payload type names; rehydrate through a fixed allow-list only.", + "evidence": "src/System.Management.Automation/engine/serialization.cs::DeserializingTypeConverter@L6670-L7678.", + "dev": null, + "doc": "REM-CLIXML-001", + "file": "05-remoting/serialization-clixml.md", + "line": 165 + }, + { + "id": "LSN-CLIXML-03", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Prohibit DTDs, cap entity expansion and nesting depth in every XML or structured reader.", + "evidence": "src/System.Management.Automation/engine/serialization.cs::InternalDeserializer.GetXmlReaderSettingsForCliXml@L4172-L4191.", + "dev": null, + "doc": "REM-CLIXML-001", + "file": "05-remoting/serialization-clixml.md", + "line": 166 + }, + { + "id": "LSN-CLIXML-04", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Add an explicit maximum input size and a streaming limit to the primary reader, and mark sensitive properties so depth-1 bags cannot leak them. DEV: DEV-SER-215.", + "evidence": "src/System.Management.Automation/engine/serialization.cs::InternalSerializer.GetDepthOfSerialization@L2322-L2381.", + "dev": "DEV-SER-215", + "doc": "REM-CLIXML-001", + "file": "05-remoting/serialization-clixml.md", + "line": 167 + }, + { + "id": "LSN-CLIXML-05", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Keep a marker on deserialized objects (type-name prefix and flag) so formatting and policy can tell data from live objects.", + "evidence": "src/System.Management.Automation/engine/serialization.cs::Deserializer@L493-L819.", + "dev": null, + "doc": "REM-CLIXML-001", + "file": "05-remoting/serialization-clixml.md", + "line": 168 + }, + { + "id": "LSN-TRANS-01", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Do not implement WSMan, Hyper-V socket or container transports; they are Windows-specific and tied to native components. DEV: DEV-REM-211.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/WSManNativeAPI.cs@L1-L3105.", + "dev": "DEV-REM-211", + "doc": "REM-TRANSPORT-001", + "file": "05-remoting/transports.md", + "line": 178 + }, + { + "id": "LSN-TRANS-02", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "If remoting is added, start with an SSH-subsystem or local-pipe transport that delegates authentication and encryption to an existing secure channel.", + "evidence": "src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs::SSHConnectionInfo@L1928-L2942.", + "dev": null, + "doc": "REM-TRANSPORT-001", + "file": "05-remoting/transports.md", + "line": 179 + }, + { + "id": "LSN-TRANS-03", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Local pipes bound to the creating user, single instance, remote clients rejected.", + "evidence": "src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs::NamedPipeUtils@L25-L156.", + "dev": null, + "doc": "REM-TRANSPORT-001", + "file": "05-remoting/transports.md", + "line": 180 + }, + { + "id": "LSN-TRANS-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Use length-prefixed binary frames instead of Base64 XML elements on stdio, keeping acknowledgement and signal semantics.", + "evidence": "src/System.Management.Automation/engine/remoting/fanin/OutOfProcTransportManager.cs::OutOfProcessUtils@L37-L403.", + "dev": null, + "doc": "REM-TRANSPORT-001", + "file": "05-remoting/transports.md", + "line": 181 + }, { "id": "LSN-TOK-01", "verdict": "REPLICATE", @@ -2320,6 +2518,270 @@ "file": "02-language/TIERING-POLICY.md", "line": 150 }, + { + "id": "LSN-HOSTCON-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Keep the exit-code rule and the specific codes (0, 1, 64, 70, 149) so automation behaves the same.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoRunspaceLoop@L1643-L1706.", + "dev": null, + "doc": "HOST-CONSOLE-001", + "file": "04-hosting/console-host.md", + "line": 181 + }, + { + "id": "LSN-HOSTCON-02", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Four-profile order and per-profile fault isolation; `-NoProfile` skips all.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoRunspaceInitialization@L1882-L2159.", + "dev": null, + "doc": "HOST-CONSOLE-001", + "file": "04-hosting/console-host.md", + "line": 182 + }, + { + "id": "LSN-HOSTCON-03", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Keep line editing external, but define a small editor interface (read line with completion and history) instead of loading a module by name.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.DoCreateRunspace@L1766-L1866.", + "dev": null, + "doc": "HOST-CONSOLE-001", + "file": "04-hosting/console-host.md", + "line": 183 + }, + { + "id": "LSN-HOSTCON-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Separate the executable into a thin launcher and a library that exposes `Start(args)` and `Start(initialSessionState, args)` like `ConsoleShell`.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleShell.cs::ConsoleShell@L15-L80.", + "dev": null, + "doc": "HOST-CONSOLE-001", + "file": "04-hosting/console-host.md", + "line": 184 + }, + { + "id": "LSN-HOSTCON-05", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Apply policy (lockdown, language mode, allowed switches) from a signed policy source before parsing switches that widen access, and refuse to start server modes without an authenticated transport. DEV: DEV-HOST-200.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs::ConsoleHost.Start@L100-L344.", + "dev": "DEV-HOST-200", + "doc": "HOST-CONSOLE-001", + "file": "04-hosting/console-host.md", + "line": 185 + }, + { + "id": "LSN-HOSTCON-06", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not carry the Windows-only pieces (`ConsoleControl` Win32 calls, jump list, `-windowstyle`, `-sta` by default, Hyper-V socket modes) into the first version. DEV: DEV-HOST-201.", + "evidence": "src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleControl.cs::ConsoleControl@L47-L3188.", + "dev": "DEV-HOST-201", + "doc": "HOST-CONSOLE-001", + "file": "04-hosting/console-host.md", + "line": 186 + }, + { + "id": "LSN-HOSTABS-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Keep the three-level host contract (host, UI, raw UI) and the lifecycle notifications; host authors depend on them.", + "evidence": "src/System.Management.Automation/engine/hostifaces/MshHost.cs::PSHost@L37-L287.", + "dev": null, + "doc": "HOST-ABSTRACTION-001", + "file": "04-hosting/host-abstraction.md", + "line": 171 + }, + { + "id": "LSN-HOSTABS-02", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Replace exception-based capability probing with explicit capability flags on the host (supports raw UI, VT, interactive session, debugger).", + "evidence": "src/System.Management.Automation/engine/hostifaces/MshHost.cs::PSHost@L37-L287.", + "dev": null, + "doc": "HOST-ABSTRACTION-001", + "file": "04-hosting/host-abstraction.md", + "line": 172 + }, + { + "id": "LSN-HOSTABS-03", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Nested prompts and debugger prompts run at the strictest language mode seen.", + "evidence": "src/System.Management.Automation/engine/hostifaces/InternalHost.cs::InternalHost.EnterNestedPrompt@L230-L361.", + "dev": null, + "doc": "HOST-ABSTRACTION-001", + "file": "04-hosting/host-abstraction.md", + "line": 173 + }, + { + "id": "LSN-HOSTABS-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Provide a headless default host and typed prompt results as part of the SDK so embedders do not subclass the full UI contract.", + "evidence": "src/System.Management.Automation/engine/hostifaces/DefaultHost.cs::DefaultHost@L17-L134.", + "dev": null, + "doc": "HOST-ABSTRACTION-001", + "file": "04-hosting/host-abstraction.md", + "line": 174 + }, + { + "id": "LSN-HOSTABS-05", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "A restricted-session baseline factory (empty, no language, fail on open error).", + "evidence": "src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.CreateRestricted@L1378-L1387.", + "dev": null, + "doc": "HOST-ABSTRACTION-001", + "file": "04-hosting/host-abstraction.md", + "line": 175 + }, + { + "id": "LSN-HOSTSDK-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "A fluent command builder (command, script, parameter, statement) with sync, async and streaming-input invocation.", + "evidence": "src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PowerShell@L575-L5954.", + "dev": null, + "doc": "HOST-SDK-001", + "file": "04-hosting/sdk.md", + "line": 166 + }, + { + "id": "LSN-HOSTSDK-02", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Add cancellation tokens and per-invocation time and memory budgets to the invoke API (the original has `Stop` only).", + "evidence": "src/System.Management.Automation/engine/hostifaces/PowerShell.cs::PSInvocationSettings@L313-L415.", + "dev": null, + "doc": "HOST-SDK-001", + "file": "04-hosting/sdk.md", + "line": 167 + }, + { + "id": "LSN-HOSTSDK-03", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Make the default embedded session restricted and require an explicit opt-in to full language and unrestricted command visibility. DEV: DEV-SDK-205.", + "evidence": "src/System.Management.Automation/engine/InitialSessionState.cs::InitialSessionState.CreateRestricted@L1378-L1387.", + "dev": "DEV-SDK-205", + "doc": "HOST-SDK-001", + "file": "04-hosting/sdk.md", + "line": 168 + }, + { + "id": "LSN-HOSTSDK-04", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Publish a written compatibility contract that classifies changes and requires a protocol version bump for wire changes.", + "evidence": "docs/dev-process/breaking-change-contract.md@L1-L60.", + "dev": null, + "doc": "HOST-SDK-001", + "file": "04-hosting/sdk.md", + "line": 169 + }, + { + "id": "LSN-HOSTSDK-05", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Ship a metapackage that pins transitive dependencies, but generate it from the build graph and publish reference assemblies for the public surface.", + "evidence": "src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj@L1-L40.", + "dev": null, + "doc": "HOST-SDK-001", + "file": "04-hosting/sdk.md", + "line": 170 + }, + { + "id": "LSN-NAT-01", + "verdict": "REDESIGN", + "area": "NLD-PLAT", + "text": "Keep one small native surface per OS with generated bindings and a layout check in CI, instead of hand-copied structs.", + "evidence": "src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122.", + "dev": "DEV-PLAT-231", + "doc": "PLAT-NATIVE-001", + "file": "06-platform/native-interop.md", + "line": 146 + }, + { + "id": "LSN-NAT-02", + "verdict": "REPLICATE", + "area": "NLD-PLAT", + "text": "One file per native function, source-generated marshalling, `SafeHandle` for owned handles, UTF-16 on Windows and UTF-8 on Unix.", + "evidence": "src/System.Management.Automation/engine/Interop/Windows/FindFirstFile.cs.", + "dev": null, + "doc": "PLAT-NATIVE-001", + "file": "06-platform/native-interop.md", + "line": 147 + }, + { + "id": "LSN-NAT-03", + "verdict": "ADAPT", + "area": "NLD-PLAT", + "text": "Pick one interop style (source-generated) and one `bool` marshalling convention per class; do not let three generations coexist.", + "evidence": "src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.NativeMethods@L935-L1122.", + "dev": null, + "doc": "PLAT-NATIVE-001", + "file": "06-platform/native-interop.md", + "line": 148 + }, + { + "id": "LSN-NAT-04", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Native library resolution must prefer the application directory and a signed or pinned per-RID folder, and refuse writable locations unless a policy allows them. DEV: DEV-PLAT-232.", + "evidence": "src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs::PowerShellAssemblyLoadContext.NativeDllHandler@L243-L250.", + "dev": "DEV-PLAT-232", + "doc": "PLAT-NATIVE-001", + "file": "06-platform/native-interop.md", + "line": 149 + }, + { + "id": "LSN-PLAT-01", + "verdict": "REDESIGN", + "area": "NLD-PLAT", + "text": "Isolate OS differences behind one platform interface (paths, process launch, links, console, logging, identity) with a runtime capability table, and keep conditional compilation for the interface implementations only.", + "evidence": "src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform@L16-L1124.", + "dev": "DEV-PLAT-230", + "doc": "PLAT-SPLIT-001", + "file": "06-platform/windows-unix-split.md", + "line": 193 + }, + { + "id": "LSN-PLAT-02", + "verdict": "REPLICATE", + "area": "NLD-PLAT", + "text": "Honour XDG directories on Unix and known folders on Windows, create on demand, and report (not hide) a failure to create them.", + "evidence": "src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.SelectProductNameForDirectory@L317-L390.", + "dev": null, + "doc": "PLAT-SPLIT-001", + "file": "06-platform/windows-unix-split.md", + "line": 194 + }, + { + "id": "LSN-PLAT-03", + "verdict": "REDESIGN", + "area": "NLD-SEC", + "text": "Define the security-zone and execution-policy concept for every OS the shell supports, or declare it unsupported through one status API. DEV: DEV-PLAT-233.", + "evidence": "src/System.Management.Automation/security/SecurityManager.cs@L416-L433.", + "dev": "DEV-PLAT-233", + "doc": "PLAT-SPLIT-001", + "file": "06-platform/windows-unix-split.md", + "line": 195 + }, + { + "id": "LSN-PLAT-04", + "verdict": "ADAPT", + "area": "NLD-PLAT", + "text": "Replace unsynchronized static caches for lazily resolved OS data with a concurrent map or a per-call lookup.", + "evidence": "src/System.Management.Automation/CoreCLR/CorePsPlatform.cs::Platform.Unix.CommonStat@L587-L745.", + "dev": null, + "doc": "PLAT-SPLIT-001", + "file": "06-platform/windows-unix-split.md", + "line": 196 + }, { "id": "LSN-CIM-01", "verdict": "AVOID", diff --git a/docs-out/data/subject-keys.json b/docs-out/data/subject-keys.json index ebf4638..77ad412 100644 --- a/docs-out/data/subject-keys.json +++ b/docs-out/data/subject-keys.json @@ -107,6 +107,18 @@ "pipeline.state.pipeline_state_members": "", "pipeline.thread.stack_size_bytes": "", "pipelineresult.bool": "", + "platform.cache_dir_unix_default": "", + "platform.getfolderpath_unix_supported": "", + "platform.interop_windows_files": "", + "platform.native_lib_name_unix": "", + "platform.native_libraryimport_count": "", + "platform.native_package": "", + "platform.native_rid_folders": "", + "platform.pinvoke_dll_name_constants": "", + "platform.shared_modules_unix": "", + "platform.unix_nonzero_exit_hex_width": "", + "platform.unix_origin_check": "", + "platform.unix_set_execution_policy": "", "provider.hierarchy": "", "provider.instance_per_operation": "", "repeat.restricted_cap": "", diff --git a/docs-out/tools/annotations/T3-docs.json b/docs-out/tools/annotations/T3-docs.json index fa0f6e4..06474b7 100644 --- a/docs-out/tools/annotations/T3-docs.json +++ b/docs-out/tools/annotations/T3-docs.json @@ -264,5 +264,168 @@ "src/Microsoft.PowerShell.Commands.Utility/commands/utility/RegisterObjectEventCommand.cs", "src/Microsoft.PowerShell.Commands.Utility/commands/utility/NewEventCommand.cs" ] + }, + "T41": { + "doc": "04-hosting/host-abstraction.md", + "pester": [ + "Host/HostUtilities", + "Utility/Read-Host", + "Utility/Write-Host", + "Core/Out-Host", + "engine/Api/InitialSessionState" + ], + "xunit": [ + "test_Runspace" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/hostifaces/MshHost.cs", + "src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs", + "src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs", + "src/System.Management.Automation/engine/hostifaces/InternalHost", + "src/System.Management.Automation/engine/InitialSessionState.cs" + ] + }, + "T42": { + "doc": "04-hosting/console-host.md", + "pester": [ + "Host/ConsoleHost", + "Host/Startup", + "Host/Logging", + "Host/Base-Directory", + "Host/PSVersionTable", + "PSReadLine", + "Host/TabCompletion/TabCompletion" + ], + "xunit": [ + "test_CommandLineParser" + ], + "sec_prefix": [ + "src/Microsoft.PowerShell.ConsoleHost/" + ] + }, + "T43": { + "doc": "04-hosting/sdk.md", + "pester": [ + "SDK/", + "engine/Api/PSCommand", + "engine/Api/BasicEngine", + "engine/Api/TaskBasedAsyncPowerShellAPI", + "engine/Api/Serialization" + ], + "xunit": [ + "test_PowerShellAPI", + "test_SessionState", + "test_PSObject" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/hostifaces/PowerShell.cs", + "src/System.Management.Automation/engine/hostifaces/ConnectionFactory.cs", + "src/System.Management.Automation/engine/hostifaces/Connection.cs", + "src/Microsoft.PowerShell.SDK/" + ] + }, + "T44": { + "doc": "05-remoting/psrp-protocol.md", + "pester": [ + "Remoting/", + "engine/Remoting" + ], + "xunit": [ + "test_RemoteHyperV" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/remoting/common/", + "src/System.Management.Automation/engine/remoting/fanin/", + "src/System.Management.Automation/engine/remoting/server/" + ] + }, + "T45": { + "doc": "05-remoting/transports.md", + "pester": [ + "Remoting/", + "engine/Remoting", + "SSHRemoting" + ], + "xunit": [ + "test_NamedPipe", + "test_RemoteHyperV" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/remoting/fanin/WSMan", + "src/System.Management.Automation/engine/remoting/common/RemoteSessionNamedPipe.cs", + "src/System.Management.Automation/engine/remoting/common/RunspaceConnectionInfo.cs", + "src/System.Management.Automation/engine/remoting/server/OutOfProc" + ] + }, + "T46": { + "doc": "05-remoting/serialization-clixml.md", + "pester": [ + "Serialization", + "Export-Clixml", + "Import-Clixml", + "CliXml", + "Clixml" + ], + "xunit": [ + "test_Serialization" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/serialization.cs", + "src/System.Management.Automation/engine/SerializationStrings.cs" + ] + }, + "T47": { + "doc": "05-remoting/jea.md", + "pester": [ + "SessionConfiguration", + "PSSessionConfiguration", + "RoleCapab", + "JEA", + "ConfigurationFile" + ], + "xunit": [], + "sec_prefix": [ + "src/System.Management.Automation/engine/remoting/fanin/InitialSessionStateProvider.cs", + "src/System.Management.Automation/engine/remoting/commands/CustomShellCommands.cs", + "src/System.Management.Automation/engine/remoting/fanin/PSPrincipal.cs" + ] + }, + "T48": { + "doc": "06-platform/windows-unix-split.md", + "pester": [ + "FileSystem.Tests", + "NativeCommandProcessor", + "NativeLinuxCommands", + "NativeUnixGlobbing", + "Base-Directory", + "ExecutionPolicy.Tests" + ], + "xunit": [ + "test_CorePsPlatform", + "test_FileSystemProvider", + "test_PSConfiguration" + ], + "sec_prefix": [ + "src/System.Management.Automation/CoreCLR/CorePsPlatform.cs", + "src/System.Management.Automation/security/SecurityManager.cs", + "src/System.Management.Automation/security/SecuritySupport.cs" + ] + }, + "T49": { + "doc": "06-platform/native-interop.md", + "pester": [ + "Assembly.LoadNative", + "NativeCommandBytePiping" + ], + "xunit": [ + "test_NativeInterop", + "test_CorePsPlatform" + ], + "sec_prefix": [ + "src/System.Management.Automation/engine/Interop/", + "src/System.Management.Automation/CoreCLR/CorePsAssemblyLoadContext.cs", + "src/System.Management.Automation/utils/PlatformInvokes.cs", + "src/System.Management.Automation/utils/tracing/SysLogProvider.cs" + ] } } \ No newline at end of file