From 5272fe21a87e8496487a2b13879c540d18465b6c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:37:34 +0000 Subject: [PATCH 1/2] Phase 2 T26, T28, T29, T210, T211: binders, runtime operations, types/operators/conversions, scope and variables, script blocks, language features, extension guide, tooling Adds BINDER-CATALOG, RUNTIME-OPERATIONS, 02.6-types-and-operators, OPERATOR-TABLE, CONVERSION-RULES, SCOPE-AND-VARIABLES, execution-context-and-session-state, SCRIPT-BLOCKS, 02.7-language-features, HOWTO-extend-language, 02.8-language-tooling; tools gen_T26/T28/T29/T210; CF-SCP-001, CF-OP-001, CF-CNV-001, CF-PIP-001; 8 DEVs; CON-011/012; VB-017..023; decision D-019; stale 'planned' links updated. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01HnR7RpZuh9mvNMRTtzxTvk --- docs-out/.work/cf/T210.json | 23 + docs-out/.work/cf/T211.json | 14 + docs-out/.work/cf/T26.json | 23 + docs-out/.work/cf/T28.json | 54 ++ docs-out/.work/cf/T29.json | 41 ++ docs-out/.work/claims/T26.jsonl | 6 + docs-out/.work/claims/T28.jsonl | 16 + docs-out/.work/claims/T29.jsonl | 11 + docs-out/.work/files-ledger.csv | 40 +- docs-out/.work/ledger-proposals/T26_T211.csv | 21 + docs-out/.work/numbers-proposals/T210.json | 9 + docs-out/.work/numbers-proposals/T26.json | 30 + docs-out/.work/numbers-proposals/T28.json | 16 + docs-out/.work/numbers-proposals/T29.json | 16 + docs-out/02-language/02.1-tokenizer.md | 2 +- docs-out/02-language/02.2-parser.md | 8 +- docs-out/02-language/02.3-ast.md | 2 +- .../02-language/02.5-compiler-and-engine.md | 2 +- .../02-language/02.6-types-and-operators.md | 138 +++++ .../02-language/02.7-language-features.md | 142 +++++ docs-out/02-language/02.8-language-tooling.md | 128 +++++ docs-out/02-language/BINDER-CATALOG.md | 83 +++ docs-out/02-language/CONVERSION-RULES.md | 202 +++++++ docs-out/02-language/EQUIVALENCE-SPEC.md | 2 +- docs-out/02-language/GRAMMAR.ebnf.md | 2 +- docs-out/02-language/HOWTO-extend-language.md | 152 +++++ docs-out/02-language/IR-SPEC.md | 4 +- docs-out/02-language/OPERATOR-TABLE.md | 139 +++++ docs-out/02-language/RUNTIME-OPERATIONS.md | 193 +++++++ docs-out/02-language/SCOPE-AND-VARIABLES.md | 241 ++++++++ docs-out/02-language/SCRIPT-BLOCKS.md | 119 ++++ .../execution-context-and-session-state.md | 125 +++++ docs-out/09-synthesis/deviation-register.md | 8 + docs-out/CONTRADICTIONS.md | 2 + docs-out/DECISIONS.md | 1 + docs-out/PROGRESS.md | 12 +- docs-out/VERIFICATION-BACKLOG.md | 7 + docs-out/data/canonical-facts.json | 52 +- docs-out/data/claims/T26.jsonl | 6 + docs-out/data/claims/T28.jsonl | 16 + docs-out/data/claims/T29.jsonl | 11 + docs-out/data/lessons.json | 517 ++++++++++++++++++ docs-out/data/numbers-extra.json | 63 +++ docs-out/data/numbers.json | 81 +++ docs-out/data/subject-keys.json | 35 +- docs-out/tools/annotations/T26-binders.json | 135 +++++ docs-out/tools/annotations/T28-operators.json | 48 ++ docs-out/tools/gen_T210.py | 16 + docs-out/tools/gen_T26.py | 113 ++++ docs-out/tools/gen_T28.py | 134 +++++ docs-out/tools/gen_T29.py | 53 ++ 51 files changed, 3263 insertions(+), 51 deletions(-) create mode 100644 docs-out/.work/cf/T210.json create mode 100644 docs-out/.work/cf/T211.json create mode 100644 docs-out/.work/cf/T26.json create mode 100644 docs-out/.work/cf/T28.json create mode 100644 docs-out/.work/cf/T29.json create mode 100644 docs-out/.work/claims/T26.jsonl create mode 100644 docs-out/.work/claims/T28.jsonl create mode 100644 docs-out/.work/claims/T29.jsonl create mode 100644 docs-out/.work/ledger-proposals/T26_T211.csv create mode 100644 docs-out/.work/numbers-proposals/T210.json create mode 100644 docs-out/.work/numbers-proposals/T26.json create mode 100644 docs-out/.work/numbers-proposals/T28.json create mode 100644 docs-out/.work/numbers-proposals/T29.json create mode 100644 docs-out/02-language/02.6-types-and-operators.md create mode 100644 docs-out/02-language/02.7-language-features.md create mode 100644 docs-out/02-language/02.8-language-tooling.md create mode 100644 docs-out/02-language/BINDER-CATALOG.md create mode 100644 docs-out/02-language/CONVERSION-RULES.md create mode 100644 docs-out/02-language/HOWTO-extend-language.md create mode 100644 docs-out/02-language/OPERATOR-TABLE.md create mode 100644 docs-out/02-language/RUNTIME-OPERATIONS.md create mode 100644 docs-out/02-language/SCOPE-AND-VARIABLES.md create mode 100644 docs-out/02-language/SCRIPT-BLOCKS.md create mode 100644 docs-out/03-runtime/execution-context-and-session-state.md create mode 100644 docs-out/data/claims/T26.jsonl create mode 100644 docs-out/data/claims/T28.jsonl create mode 100644 docs-out/data/claims/T29.jsonl create mode 100644 docs-out/tools/annotations/T26-binders.json create mode 100644 docs-out/tools/annotations/T28-operators.json create mode 100644 docs-out/tools/gen_T210.py create mode 100644 docs-out/tools/gen_T26.py create mode 100644 docs-out/tools/gen_T28.py create mode 100644 docs-out/tools/gen_T29.py diff --git a/docs-out/.work/cf/T210.json b/docs-out/.work/cf/T210.json new file mode 100644 index 0000000..455b0f1 --- /dev/null +++ b/docs-out/.work/cf/T210.json @@ -0,0 +1,23 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-SB-090", + "baseline_behavior": "ScriptBlock can become a PowerShell object and a steppable pipeline.", + "proposed_behavior": "Omitted initially.", + "benefit": "Less surface.", + "compatibility_cost": "No GetPowerShell/GetSteppablePipeline.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-FEAT-091", + "baseline_behavior": "workflow/parallel/sequence/inlinescript parse and then fail in semantic checks; from/define/var are reserved.", + "proposed_behavior": "Reject at parse time.", + "benefit": "No dead syntax.", + "compatibility_cost": "Different error text.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T211.json b/docs-out/.work/cf/T211.json new file mode 100644 index 0000000..c8439cc --- /dev/null +++ b/docs-out/.work/cf/T211.json @@ -0,0 +1,14 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-TOOLS-100", + "baseline_behavior": "PSParser.Tokenize returns PowerShell 2 style tokens by rewriting the saved token list.", + "proposed_behavior": "Only the modern token list.", + "benefit": "No V2 branches in the parser.", + "compatibility_cost": "Legacy tools unsupported.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T26.json b/docs-out/.work/cf/T26.json new file mode 100644 index 0000000..7c0dca2 --- /dev/null +++ b/docs-out/.work/cf/T26.json @@ -0,0 +1,23 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-BIND-050", + "baseline_behavior": "Member access goes through COM and ETS-adapter aware binders.", + "proposed_behavior": "Plain reflection member lookup; no COM binders.", + "benefit": "Smaller binder layer.", + "compatibility_cost": "No COM objects.", + "affected_cfs": [], + "status": "PROPOSED" + }, + { + "id": "DEV-RTOPS-051", + "baseline_behavior": "Classes are emitted with Reflection.Emit and method bodies wrapped by ScriptBlockMemberMethodWrapper.", + "proposed_behavior": "Classes are interpreter objects, not CLR types.", + "benefit": "No Reflection.Emit dependency.", + "compatibility_cost": "Classes are not usable from other .NET code.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T28.json b/docs-out/.work/cf/T28.json new file mode 100644 index 0000000..1e12e10 --- /dev/null +++ b/docs-out/.work/cf/T28.json @@ -0,0 +1,54 @@ +{ + "cfs": [ + { + "id": "CF-OP-001", + "subject": "operator precedence and associativity", + "statement": "**Operator precedence and associativity.** Binary operators are parsed by precedence climbing and **all are left-associative**; from lowest to highest precedence the levels are: logical (`-and`, `-or`, `-xor`) = 1; bitwise (`-band`, `-bor`, `-bxor`) = 2; comparison group (all `-eq`/`-ne`/`-gt`/`-ge`/`-lt`/`-le`, `-like`, `-match`, `-replace`, `-contains`, `-in` families with their `i`/`c` variants, `-is`, `-isnot`, `-as`, `-split`, `-join`, `-shl`, `-shr`) = 5; null-coalescing `??` = 7; additive (`+`, `-`) = 9; multiplicative (`*`, `/`, `%`) = 10; format `-f` = 12; range `..` = 13. The comma (array-literal) operator binds tighter than every binary operator; the unary operators, casts and `++`/`--` bind tighter than the comma; the ternary `?:` (right-nested) and assignment (`=`, `+=`, `-=`, `*=`, `/=`, `%=`, `??=`, parsed at statement level with a statement on the right) bind looser than all binary operators; `&&` and `||` chain pipelines and are not expression operators. Operators spelled with a leading dash are recognized as operators only when the tokenizer is in expression mode.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950", + "src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752" + ], + "owner": "02-language/OPERATOR-TABLE.md", + "status": "open" + }, + { + "id": "CF-CNV-001", + "subject": "conversion algorithm and converter order", + "statement": "**Conversion algorithm and converter order.** For a source type S and target type T the engine returns the first applicable converter in this order (cached per type pair): (1) a cached entry; (2) if S is the null type: `Nullable<\u2026>` \u2192 null, any reference type \u2192 null, a value type \u2192 no conversion (numeric, `char`, `string`, `bool`, `PSReference`, `switch` and `void` targets have pre-cached null conversions); (3) if T is assignable from S: identity (same type) or assignable; (4) by-ref-like types: none; (5) S is a `PSObject`: none (the caller unwraps first); (6) T is `PSObject`: wrap; (7) under Constrained Language Mode: only targets `object`, `object[]` or the core type list are converted (others are refused, or only audited in WDAC audit mode); (8) **language conversions**: `Nullable` through U, `void`, `bool` (from `IList`, enum, value type, class), `string`, arrays (related arrays element-wise, unrelated arrays per element, enumerable to array, scalar to one-element array), `Array`, `Hashtable` from `IDictionary`, `PSReference`, `XmlDocument`, `StringCollection`, `ScriptBlock` to delegate, `PSCustomObject`, integer to enum, `PSMethod` to delegate; (9) a static `Parse` method on T; (10) a static `Create` method; (11) a constructor of T taking S; (12) a cast operator (implicit, then explicit) on S or T; (13) `IConvertible` (numeric sources through the string-convertible path, other `IConvertible` sources via `Convert`); (14) for an `IDictionary` source: T's no-argument constructor then property assignment from the keys; (15) an `IEnumerable` constructor; (16) property-based conversion; if S or T may have a `TypeConverter`/`PSTypeConverter`, or a value-dependent converter was found, the result is wrapped to try the custom/value-dependent converter first and the found converter second; (17) otherwise no conversion (rank `None`, a failing converter that raises `PSInvalidCastException`). Numeric-to-numeric, number-to-string, number-to-bool and null-to-number conversions are pre-cached with fixed ranks.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828", + "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureLanguageConversion@L4966-L5120" + ], + "owner": "02-language/CONVERSION-RULES.md", + "status": "open" + }, + { + "id": "CF-PIP-001", + "subject": "output unrolling and enumeration rules", + "statement": "**Output unrolling and enumeration rules.** A value is *enumerated* (treated as a sequence) when, after unwrapping `PSObject`: it is `AutomationNull` (empty sequence), an array, a `DataTable` (its rows), a COM enumerable, any other `IEnumerable` that is not a string, `IDictionary` or `XmlNode` (the generic `IEnumerable` enumerator is preferred), or an `IEnumerator`; `null`, strings, dictionaries (hashtables), `XmlNode` and all other objects are **not** enumerated and are single values. When a statement writes its value to the output pipe: `AutomationNull` writes nothing; a non-enumerable value is written as one object (mutable value types are copied first); an enumerable value is **unrolled exactly one level**: each element is written separately (elements that are themselves collections stay whole), and an enumerator is disposed afterwards if the engine created it.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L698", + "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipeWriterBinder@L841-L926", + "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::EnumerableOps.WriteEnumerableToPipe@L3623-L3640" + ], + "owner": "02-language/02.6-types-and-operators.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-TYP-070", + "baseline_behavior": "Constrained Language Mode restricts conversion targets inside FigureConversion.", + "proposed_behavior": "No language mode in the conversion layer; allow-list at the type-system boundary.", + "benefit": "Simpler conversion code.", + "compatibility_cost": "No Constrained Language Mode.", + "affected_cfs": [ + "CF-CNV-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T29.json b/docs-out/.work/cf/T29.json new file mode 100644 index 0000000..5a1985a --- /dev/null +++ b/docs-out/.work/cf/T29.json @@ -0,0 +1,41 @@ +{ + "cfs": [ + { + "id": "CF-SCP-001", + "subject": "scope, variable lookup and assignment rules", + "statement": "**Scope, lookup and assignment rules.** A variable name is parsed into an optional scope qualifier (`global:`, `local:`, `script:`, `private:`) or a provider drive (`name:`), and an unqualified name. **Reading:** an unqualified name is searched in the current scope and then in each parent scope up to the global scope, and the first match wins; a variable marked `Private` is skipped when it is found in any scope other than the current one; `global:`, `local:`/`private:` and `script:` search exactly one scope (the global scope, the current scope, the script scope respectively); inside one scope the compiled locals tuple (and any dot-sourced tuples) is checked before the variable table; a drive-qualified name is read through the provider. A name that is not found yields `$null`, or a `VariableIsUndefined` error under strict mode (version 2: always; version 1: except inside expandable strings). **Writing:** an unqualified or `local:`/`private:` assignment always targets the current scope (a parent's variable of the same name is never modified; a new local variable is created if the current scope has none), `script:` and `global:` target the script and global scope; assigning to an existing variable in the target scope keeps its object and replaces its value after running its attributes (transformation, then validation), and a `ReadOnly` or `Constant` variable cannot be written (`VariableNotWritable`; `-Force` can overwrite `ReadOnly` but never `Constant`). **New scopes:** every function or script block call (not dot-sourcing) creates a child scope whose parent is the caller's scope; the script scope is the nearest enclosing scope created for a script file, or the global scope; variables marked `AllScope` are shared by reference with every new scope. **Dot-sourcing** creates no scope: the callee's locals tuple is pushed onto the current scope's `DottedScopes` for the duration of the call.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297", + "src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.SetVariable@L401-L518", + "src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L19-L120", + "src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal.NewScope@L227-L240", + "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L117-L143" + ], + "owner": "02-language/SCOPE-AND-VARIABLES.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-SCP-080", + "baseline_behavior": "Values assigned to global variables are marked untrusted in Constrained Language Mode.", + "proposed_behavior": "No trust marking.", + "benefit": "Simpler assignment path.", + "compatibility_cost": "No Constrained Language Mode.", + "affected_cfs": [ + "CF-SCP-001" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-CTX-081", + "baseline_behavior": "ExecutionContext carries language mode and invalidates binder caches on change.", + "proposed_behavior": "No language modes.", + "benefit": "No cache invalidation logic.", + "compatibility_cost": "No restricted modes.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/claims/T26.jsonl b/docs-out/.work/claims/T26.jsonl new file mode 100644 index 0000000..414e9d0 --- /dev/null +++ b/docs-out/.work/claims/T26.jsonl @@ -0,0 +1,6 @@ +{"id": "CLM-RTOPS-0001", "subject_key": "erroraction.default", "predicate": "ErrorActionPreference default in statement error handling", "value": "Continue", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps@L1460-L2197"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0002", "subject_key": "erroraction.throw_statement", "predicate": "exception from a throw statement without traps", "value": "rethrown", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps@L1460-L2197"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0003", "subject_key": "trap.continue", "predicate": "continue inside a trap", "value": "SilentlyContinue", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps@L1460-L2197"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0004", "subject_key": "loopflow.empty_label", "predicate": "empty break/continue label", "value": "matches any loop", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/lang/parserutils.cs::LoopFlowException@L36-L67"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0005", "subject_key": "hashtable.literal.duplicate", "predicate": "duplicate or null key in hashtable literal", "value": "runtime error", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::HashtableOps@L1402-L1458"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0006", "subject_key": "binder.count", "predicate": "binder classes in dataset", "value": "30", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422"], "doc": "LANG-RTOPS-001", "cf": null} diff --git a/docs-out/.work/claims/T28.jsonl b/docs-out/.work/claims/T28.jsonl new file mode 100644 index 0000000..c8c48c6 --- /dev/null +++ b/docs-out/.work/claims/T28.jsonl @@ -0,0 +1,16 @@ +{"id": "CLM-TYP-0001", "subject_key": "enumeration.string", "predicate": "string is enumerated by PSEnumerableBinder", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L698"], "doc": "LANG-TYPES-001", "cf": "CF-PIP-001"} +{"id": "CLM-TYP-0002", "subject_key": "enumeration.dictionary", "predicate": "IDictionary is enumerated", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L698"], "doc": "LANG-TYPES-001", "cf": "CF-PIP-001"} +{"id": "CLM-TYP-0003", "subject_key": "output.unroll_depth", "predicate": "pipe write unrolls enumerables by", "value": "one level", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::EnumerableOps.WriteEnumerableToPipe@L3623-L3640"], "doc": "LANG-TYPES-001", "cf": "CF-PIP-001"} +{"id": "CLM-TYP-0004", "subject_key": "arith.int_overflow", "predicate": "int add/sub/mul overflow result type", "value": "double", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0005", "subject_key": "arith.int_divide", "predicate": "int/int result", "value": "int if exact else double", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0006", "subject_key": "arith.zero_divide", "predicate": "division by zero", "value": "RuntimeException", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0007", "subject_key": "truthiness.string", "predicate": "string truthiness", "value": "non-empty", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1018"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0008", "subject_key": "truthiness.list", "predicate": "single-element list truthiness", "value": "truthiness of the element (list element: non-empty)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1018"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0009", "subject_key": "pipelineresult.bool", "predicate": "captured pipeline result to bool", "value": "0:false, 1:element as bool, >1:true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipelineResultToBoolBinder@L1119-L1178"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0010", "subject_key": "destructuring.shape", "predicate": "array destructuring with more values than targets", "value": "last target receives the remaining values as a slice", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSArrayAssignmentRHSBinder@L935-L1042"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0011", "subject_key": "convert.double_format", "predicate": "double to string format", "value": "G15 (invariant)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives@L308-L309"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0012", "subject_key": "convert.string_to_number", "predicate": "string to number uses", "value": "Parser.ScanNumber", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.TryScanNumber@L2882-L2900"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0013", "subject_key": "convert.constrained_gate", "predicate": "constrained language conversion targets", "value": "object, object[] or CoreTypes only", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0014", "subject_key": "operator.precedence.levels", "predicate": "binary precedence levels", "value": "logical1 bitwise2 comparison5 coalesce7 add9 multiply10 format12 range13", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752"], "doc": "LANG-TYPES-001", "cf": "CF-OP-001"} +{"id": "CLM-TYP-0015", "subject_key": "repeat.restricted_cap", "predicate": "string/array repetition cap in RestrictedLanguage", "value": "1024", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/StringOps.cs::StringOps@L15-L107"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0016", "subject_key": "hashtable.add.comparer", "predicate": "hashtable + hashtable comparer", "value": "CurrentCultureIgnoreCase", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::HashtableOps@L1402-L1458"], "doc": "LANG-TYPES-001", "cf": null} diff --git a/docs-out/.work/claims/T29.jsonl b/docs-out/.work/claims/T29.jsonl new file mode 100644 index 0000000..69da0d8 --- /dev/null +++ b/docs-out/.work/claims/T29.jsonl @@ -0,0 +1,11 @@ +{"id": "CLM-SCP-0001", "subject_key": "scope.read.unqualified", "predicate": "unqualified variable read searches", "value": "current scope then parents; first match wins", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScopedItemSearcher.cs::ScopedItemSearcher@L15-L239"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0002", "subject_key": "scope.read.private_skip", "predicate": "private variable found outside the current scope", "value": "skipped", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0003", "subject_key": "scope.write.unqualified", "predicate": "unqualified assignment targets", "value": "current scope (forced local)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L19-L120"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0004", "subject_key": "scope.allscope.copy", "predicate": "AllScope variables in new scopes", "value": "same object copied by reference from the parent", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.AddSessionStateScopeDefaultVariables@L1756-L1782"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0005", "subject_key": "scope.global.defaults", "predicate": "default variables of the global scope", "value": "null, false, true (AllScope)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.AddSessionStateScopeDefaultVariables@L1756-L1782"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0006", "subject_key": "scope.automatic.slots", "predicate": "fixed automatic variable slots", "value": "9 (Underbar..PSCommandPath)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SpecialVariables.cs::AutomaticVariable@L401-L413"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0007", "subject_key": "scope.preference.slots", "predicate": "preference variable slots", "value": "8 (Debug..Progress), indexes 9-16", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SpecialVariables.cs::PreferenceVariable@L415-L425"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0008", "subject_key": "scope.strict.walk", "predicate": "strict mode lookup stops at", "value": "module scope", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L117-L143"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0009", "subject_key": "scope.strict.version1_exception", "predicate": "strict version 1 undefined variable error", "value": "not raised inside expandable strings", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps@L15-L326"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0010", "subject_key": "variablepath.drive_rule", "predicate": "path with a colon after index 0 and no scope prefix", "value": "drive qualified", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/VariablePath.cs::VariablePath@L29-L313"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0011", "subject_key": "closure.captures", "predicate": "GetNewClosure copies", "value": "current scope variables with Options None, by value", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo.CaptureLocals@L1343-L1370"], "doc": "LANG-SCOPE-001", "cf": null} diff --git a/docs-out/.work/files-ledger.csv b/docs-out/.work/files-ledger.csv index 963e495..955cdc4 100644 --- a/docs-out/.work/files-ledger.csv +++ b/docs-out/.work/files-ledger.csv @@ -1652,7 +1652,7 @@ src/System.Management.Automation/engine/EnumExpressionEvaluator.cs,644,B,,,unrea src/System.Management.Automation/engine/EnumMinimumDisambiguation.cs,127,B,,,unread src/System.Management.Automation/engine/ErrorPackage.cs,1837,B,,,unread src/System.Management.Automation/engine/EventManager.cs,2648,B,,,unread -src/System.Management.Automation/engine/ExecutionContext.cs,1711,A,,,unread +src/System.Management.Automation/engine/ExecutionContext.cs,1711,A,100-145;275-300;521-560;862-885,RT-CONTEXT-001,partial src/System.Management.Automation/engine/ExtendedTypeSystemException.cs,641,B,,,unread src/System.Management.Automation/engine/ExternalScriptInfo.cs,633,B,,,unread src/System.Management.Automation/engine/ExtraAdapter.cs,385,B,,,unread @@ -1665,7 +1665,7 @@ src/System.Management.Automation/engine/InitialSessionState.cs,5674,B,,,unread src/System.Management.Automation/engine/InternalCommands.cs,2752,B,,,unread src/System.Management.Automation/engine/InvocationInfo.cs,537,B,,,unread src/System.Management.Automation/engine/ItemCmdletProviderInterfaces.cs,2107,B,,,unread -src/System.Management.Automation/engine/LanguagePrimitives.cs,5888,A,,,unread +src/System.Management.Automation/engine/LanguagePrimitives.cs,5888,A,449-560;599-865;980-1060;1670-1770;2882-3074;3096-3135;3322-3400;3581-3660;4468-4520;4860-4895;4966-5120;5641-5848,LANG-CONV-001;LANG-TYPES-001,partial src/System.Management.Automation/engine/ManagementObjectAdapter.cs,1264,B,,,unread src/System.Management.Automation/engine/MergedCommandParameterMetadata.cs,717,A,,,unread src/System.Management.Automation/engine/MinishellParameterBinderController.cs,384,A,,,unread @@ -1708,7 +1708,7 @@ src/System.Management.Automation/engine/PseudoParameterBinder.cs,136,A,,,unread src/System.Management.Automation/engine/PseudoParameters.cs,246,B,,,unread src/System.Management.Automation/engine/QuestionMarkVariable.cs,43,B,,,unread src/System.Management.Automation/engine/ReflectionParameterBinder.cs,322,A,,,unread -src/System.Management.Automation/engine/ScopedItemSearcher.cs,487,A,,,unread +src/System.Management.Automation/engine/ScopedItemSearcher.cs,487,A,15-297,LANG-SCOPE-001,partial src/System.Management.Automation/engine/ScriptCommand.cs,16,A,,,unread src/System.Management.Automation/engine/ScriptCommandProcessor.cs,662,A,,,unread src/System.Management.Automation/engine/ScriptInfo.cs,126,B,,,unread @@ -1729,15 +1729,15 @@ src/System.Management.Automation/engine/SessionStateNavigation.cs,1750,B,,,unrea src/System.Management.Automation/engine/SessionStateProperty.cs,1118,B,,,unread src/System.Management.Automation/engine/SessionStateProviderAPIs.cs,1556,B,,,unread src/System.Management.Automation/engine/SessionStatePublic.cs,374,B,,,unread -src/System.Management.Automation/engine/SessionStateScope.cs,2013,A,,,unread -src/System.Management.Automation/engine/SessionStateScopeAPIs.cs,326,A,,,unread +src/System.Management.Automation/engine/SessionStateScope.cs,2013,A,27-110;293-345;401-520;661-700;1736-1790,LANG-SCOPE-001,partial +src/System.Management.Automation/engine/SessionStateScopeAPIs.cs,326,A,200-300,LANG-SCOPE-001,partial src/System.Management.Automation/engine/SessionStateScopeEnumerator.cs,103,A,,,unread src/System.Management.Automation/engine/SessionStateSecurityDescriptorInterface.cs,578,B,,,unread src/System.Management.Automation/engine/SessionStateStrings.cs,180,B,,,unread src/System.Management.Automation/engine/SessionStateUtils.cs,324,B,,,unread -src/System.Management.Automation/engine/SessionStateVariableAPIs.cs,1865,A,,,unread +src/System.Management.Automation/engine/SessionStateVariableAPIs.cs,1865,A,532-640;1072-1200,LANG-SCOPE-001,partial src/System.Management.Automation/engine/ShellVariable.cs,916,A,,,unread -src/System.Management.Automation/engine/SpecialVariables.cs,426,A,,,unread +src/System.Management.Automation/engine/SpecialVariables.cs,426,A,320-353;401-425,LANG-SCOPE-001,partial src/System.Management.Automation/engine/ThirdPartyAdapter.cs,365,B,,,unread src/System.Management.Automation/engine/TypeMetadata.cs,1567,B,,,unread src/System.Management.Automation/engine/TypeTable.cs,4779,B,,,unread @@ -1748,7 +1748,7 @@ src/System.Management.Automation/engine/UserFeedbackParameters.cs,213,B,,,unread src/System.Management.Automation/engine/Utils.cs,1866,B,,,unread src/System.Management.Automation/engine/VariableAttributeCollection.cs,160,A,,,unread src/System.Management.Automation/engine/VariableInterfaces.cs,448,A,,,unread -src/System.Management.Automation/engine/VariablePath.cs,322,A,,,unread +src/System.Management.Automation/engine/VariablePath.cs,322,A,1-322,LANG-SCOPE-001,read src/System.Management.Automation/engine/cmdlet.cs,1855,B,,,unread src/System.Management.Automation/engine/pipeline.cs,1668,A,,,unread src/System.Management.Automation/engine/regex.cs,1384,B,,,unread @@ -1817,7 +1817,7 @@ src/System.Management.Automation/engine/ComInterop/InteropServices/ComEventsMeth src/System.Management.Automation/engine/ComInterop/InteropServices/ComEventsSink.cs,285,B,,,unread src/System.Management.Automation/engine/ComInterop/InteropServices/IDispatch.cs,49,B,,,unread src/System.Management.Automation/engine/ComInterop/InteropServices/Variant.cs,732,B,,,unread -src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs,564,B,,,unread +src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs,564,B,60-110,LANG-TOOLING-001,partial src/System.Management.Automation/engine/CommandCompletion/CompletionAnalysis.cs,2693,B,,,unread src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs,9251,B,,,unread src/System.Management.Automation/engine/CommandCompletion/CompletionHelpers.cs,322,B,,,unread @@ -1871,7 +1871,7 @@ src/System.Management.Automation/engine/Modules/ModuleSpecification.cs,384,B,,,u src/System.Management.Automation/engine/Modules/ModuleUtils.cs,661,B,,,unread src/System.Management.Automation/engine/Modules/NewModuleCommand.cs,259,B,,,unread src/System.Management.Automation/engine/Modules/NewModuleManifestCommand.cs,1227,B,,,unread -src/System.Management.Automation/engine/Modules/PSModuleInfo.cs,1650,B,,,unread +src/System.Management.Automation/engine/Modules/PSModuleInfo.cs,1650,B,1245-1380,LANG-SCRIPTBLOCK-001,partial src/System.Management.Automation/engine/Modules/RemoteDiscoveryHelper.cs,1105,B,,,unread src/System.Management.Automation/engine/Modules/RemoveModuleCommand.cs,382,B,,,unread src/System.Management.Automation/engine/Modules/ScriptAnalysis.cs,593,B,,,unread @@ -1961,9 +1961,9 @@ src/System.Management.Automation/engine/interpreter/TypeOperations.cs,197,A,1-19 src/System.Management.Automation/engine/interpreter/Utilities.cs,1157,A,1-1157,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read src/System.Management.Automation/engine/lang/codegen.cs,110,A,,,unread src/System.Management.Automation/engine/lang/parserutils.cs,2010,A,1-2010,LANG-ENGINE-001,read -src/System.Management.Automation/engine/lang/scriptblock.cs,1479,A,,,unread +src/System.Management.Automation/engine/lang/scriptblock.cs,1479,A,83-130;397-420;518-530,LANG-SCRIPTBLOCK-001,partial src/System.Management.Automation/engine/lang/interface/PSParseError.cs,52,A,,,unread -src/System.Management.Automation/engine/lang/interface/PSParser.cs,194,A,,,unread +src/System.Management.Automation/engine/lang/interface/PSParser.cs,194,A,1-194,LANG-TOOLING-001,read src/System.Management.Automation/engine/lang/interface/PSToken.cs,644,A,,,unread src/System.Management.Automation/engine/parser/AstVisitor.cs,822,A,1-822,LANG-AST-001;LANG-ASTCAT-001,read src/System.Management.Automation/engine/parser/CharTraits.cs,414,A,1-414,LANG-TOKCAT-001;LANG-TOKENIZER-001,read @@ -1971,13 +1971,13 @@ src/System.Management.Automation/engine/parser/Compiler.cs,7082,A,1-7082,LANG-CO src/System.Management.Automation/engine/parser/ConstantValues.cs,578,A,1-578,LANG-AST-001,read src/System.Management.Automation/engine/parser/DebugViewWriter.cs,1219,A,,,unread src/System.Management.Automation/engine/parser/PSType.cs,1520,A,1-1520,LANG-SEMANTIC-001,read -src/System.Management.Automation/engine/parser/Parser.cs,8279,A,1-8279,LANG-ERRCAT-001;LANG-GRAMMAR-001;LANG-PARSER-001,read +src/System.Management.Automation/engine/parser/Parser.cs,8279,A,1-8279,LANG-ERRCAT-001;LANG-GRAMMAR-001;LANG-PARSER-001;LANG-TOOLING-001,read src/System.Management.Automation/engine/parser/Position.cs,816,A,1-816,LANG-AST-001,read src/System.Management.Automation/engine/parser/PreOrderVisitor.cs,264,A,1-264,LANG-AST-001;LANG-ASTCAT-001,read src/System.Management.Automation/engine/parser/SafeValues.cs,807,A,1-807,LANG-AST-001,read src/System.Management.Automation/engine/parser/SemanticChecks.cs,2455,A,1-2455,LANG-SEMANTIC-001,read src/System.Management.Automation/engine/parser/SymbolResolver.cs,763,A,1-763,LANG-SEMANTIC-001,read -src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs,3338,A,,,unread +src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs,3338,A,1-3338,LANG-TOOLING-001,read src/System.Management.Automation/engine/parser/TypeResolver.cs,980,A,1-980,LANG-SEMANTIC-001,read src/System.Management.Automation/engine/parser/VariableAnalysis.cs,1848,A,1-1848,LANG-SEMANTIC-001,read src/System.Management.Automation/engine/parser/ast.cs,10845,A,1-10845,LANG-AST-001;LANG-ASTCAT-001,read @@ -2082,13 +2082,13 @@ src/System.Management.Automation/engine/remoting/server/serverremotingprotocolim src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs,2553,A,1-2553,LANG-ENGINE-001,read src/System.Management.Automation/engine/runtime/MutableTuple.cs,2536,A,,,unread src/System.Management.Automation/engine/runtime/ScriptBlockToPowerShell.cs,953,A,,,unread -src/System.Management.Automation/engine/runtime/Binding/Binders.cs,8015,A,,,unread -src/System.Management.Automation/engine/runtime/Operations/ArrayOps.cs,302,A,,,unread +src/System.Management.Automation/engine/runtime/Binding/Binders.cs,8015,A,580-735;805-1042;1134-1178;2179-2242;2341-2470;2594-2870;3046-3180;3378-3420;3774-3830;5180-5230;7400-7440,LANG-BINDERCAT-001;LANG-OPTABLE-001;LANG-TYPES-001,partial +src/System.Management.Automation/engine/runtime/Operations/ArrayOps.cs,302,A,15-112,LANG-RTOPS-001,partial src/System.Management.Automation/engine/runtime/Operations/ClassOps.cs,375,A,,,unread -src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs,3765,A,,,unread -src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs,945,A,,,unread -src/System.Management.Automation/engine/runtime/Operations/StringOps.cs,108,A,,,unread -src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs,327,A,,,unread +src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs,3765,A,1402-1458;1699-1908;3623-3640,LANG-RTOPS-001,partial +src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs,945,A,15-130,LANG-TYPES-001,partial +src/System.Management.Automation/engine/runtime/Operations/StringOps.cs,108,A,15-60,LANG-RTOPS-001,partial +src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs,327,A,1-260,LANG-SCOPE-001,partial src/System.Management.Automation/help/AliasHelpInfo.cs,110,B,,,unread src/System.Management.Automation/help/AliasHelpProvider.cs,322,B,,,unread src/System.Management.Automation/help/BaseCommandHelpInfo.cs,489,B,,,unread diff --git a/docs-out/.work/ledger-proposals/T26_T211.csv b/docs-out/.work/ledger-proposals/T26_T211.csv new file mode 100644 index 0000000..114b806 --- /dev/null +++ b/docs-out/.work/ledger-proposals/T26_T211.csv @@ -0,0 +1,21 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/LanguagePrimitives.cs,449-560;599-865;980-1060;1670-1770;2882-3074;3096-3135;3322-3400;3581-3660;4468-4520;4860-4895;4966-5120;5641-5848,LANG-CONV-001;LANG-TYPES-001 +src/System.Management.Automation/engine/runtime/Binding/Binders.cs,580-735;805-1042;1134-1178;2179-2242;2341-2470;2594-2870;3046-3180;3378-3420;3774-3830;5180-5230;7400-7440,LANG-BINDERCAT-001;LANG-OPTABLE-001;LANG-TYPES-001 +src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs,15-130,LANG-TYPES-001 +src/System.Management.Automation/engine/runtime/Operations/ArrayOps.cs,15-112,LANG-RTOPS-001 +src/System.Management.Automation/engine/runtime/Operations/StringOps.cs,15-60,LANG-RTOPS-001 +src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs,1402-1458;1699-1908;3623-3640,LANG-RTOPS-001 +src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs,1-260,LANG-SCOPE-001 +src/System.Management.Automation/engine/VariablePath.cs,1-322,LANG-SCOPE-001 +src/System.Management.Automation/engine/SessionStateScope.cs,27-110;293-345;401-520;661-700;1736-1790,LANG-SCOPE-001 +src/System.Management.Automation/engine/ScopedItemSearcher.cs,15-297,LANG-SCOPE-001 +src/System.Management.Automation/engine/SessionStateVariableAPIs.cs,532-640;1072-1200,LANG-SCOPE-001 +src/System.Management.Automation/engine/SessionStateScopeAPIs.cs,200-300,LANG-SCOPE-001 +src/System.Management.Automation/engine/ExecutionContext.cs,100-145;275-300;521-560;862-885,RT-CONTEXT-001 +src/System.Management.Automation/engine/SpecialVariables.cs,320-353;401-425,LANG-SCOPE-001 +src/System.Management.Automation/engine/lang/scriptblock.cs,83-130;397-420;518-530,LANG-SCRIPTBLOCK-001 +src/System.Management.Automation/engine/Modules/PSModuleInfo.cs,1245-1380,LANG-SCRIPTBLOCK-001 +src/System.Management.Automation/engine/lang/interface/PSParser.cs,1-194,LANG-TOOLING-001 +src/System.Management.Automation/engine/parser/Parser.cs,1120-1152,LANG-TOOLING-001 +src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs,60-110,LANG-TOOLING-001 +src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs,1-3338,LANG-TOOLING-001 diff --git a/docs-out/.work/numbers-proposals/T210.json b/docs-out/.work/numbers-proposals/T210.json new file mode 100644 index 0000000..e3f5023 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T210.json @@ -0,0 +1,9 @@ +[ + { + "key": "LANGUAGE_FEATURE_ROWS", + "cmd": "python3 docs-out/tools/gen_T210.py --count rows", + "dataset": "02.7-language-features.md", + "label": "APPROX", + "notes": "T210" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T26.json b/docs-out/.work/numbers-proposals/T26.json new file mode 100644 index 0000000..6cd385a --- /dev/null +++ b/docs-out/.work/numbers-proposals/T26.json @@ -0,0 +1,30 @@ +[ + { + "key": "BINDER_CLASSES", + "cmd": "python3 docs-out/tools/gen_T26.py --count binders", + "dataset": "binders.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "BINDER_CLASSES_PS", + "cmd": "python3 docs-out/tools/gen_T26.py --count ps_binders", + "dataset": "binders.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "BINDER_CREATION_SITES", + "cmd": "python3 docs-out/tools/gen_T26.py --count created", + "dataset": "binders.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "RUNTIME_OP_TYPES", + "cmd": "python3 docs-out/tools/gen_T26.py --count optypes", + "dataset": "types.json", + "label": "APPROX", + "notes": "T26" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T28.json b/docs-out/.work/numbers-proposals/T28.json new file mode 100644 index 0000000..3c12a75 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T28.json @@ -0,0 +1,16 @@ +[ + { + "key": "OPERATOR_KINDS", + "cmd": "python3 docs-out/tools/gen_T28.py --count operators", + "dataset": "tables.json", + "label": "APPROX", + "notes": "T28" + }, + { + "key": "CONVERSION_RANKS", + "cmd": "python3 docs-out/tools/gen_T28.py --count ranks", + "dataset": "enums.json", + "label": "APPROX", + "notes": "T28" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T29.json b/docs-out/.work/numbers-proposals/T29.json new file mode 100644 index 0000000..fe73d7e --- /dev/null +++ b/docs-out/.work/numbers-proposals/T29.json @@ -0,0 +1,16 @@ +[ + { + "key": "SPECIAL_VARIABLE_CONSTANTS", + "cmd": "python3 docs-out/tools/gen_T29.py --count constants", + "dataset": "auto-variables.json", + "label": "APPROX", + "notes": "T29" + }, + { + "key": "EXECCONTEXT_MEMBERS", + "cmd": "python3 docs-out/tools/gen_T29.py --count execcontext", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "T29" + } +] \ No newline at end of file diff --git a/docs-out/02-language/02.1-tokenizer.md b/docs-out/02-language/02.1-tokenizer.md index dd225a4..fab8243 100644 --- a/docs-out/02-language/02.1-tokenizer.md +++ b/docs-out/02-language/02.1-tokenizer.md @@ -261,7 +261,7 @@ The dynamic-keyword table is `[ThreadStatic]` (`src/System.Management.Automation ## 11. Extension points and customization hooks -Add a keyword: add rows to `s_keywordText`/`s_keywordTokenKind`, a `TokenKind` member, and rows to `s_staticTokenFlags`/`s_tokenText` (the `DEBUG` assertions will fail otherwise). Add a dash operator: rows in `_operatorText`/`s_operatorTokenKind` plus the kind tables and precedence flag. Detailed recipes are in HOWTO-extend-language (planned, T210). Dynamic keywords (DSC) add keywords at run time. +Add a keyword: add rows to `s_keywordText`/`s_keywordTokenKind`, a `TokenKind` member, and rows to `s_staticTokenFlags`/`s_tokenText` (the `DEBUG` assertions will fail otherwise). Add a dash operator: rows in `_operatorText`/`s_operatorTokenKind` plus the kind tables and precedence flag. Detailed recipes are in [HOWTO-extend-language.md](HOWTO-extend-language.md). Dynamic keywords (DSC) add keywords at run time. ## 12. Dependencies diff --git a/docs-out/02-language/02.2-parser.md b/docs-out/02-language/02.2-parser.md index 46c498f..5590f62 100644 --- a/docs-out/02-language/02.2-parser.md +++ b/docs-out/02-language/02.2-parser.md @@ -22,7 +22,7 @@ generated_from: [data/parser-methods.json, data/resources.json, data/tables.json The parser turns script text into a `ScriptBlockAst`. It is a hand-written **recursive-descent parser** with a **one-token lookahead** that pulls tokens from the tokenizer on demand and tells the tokenizer which mode to use (see CF-LEX-001 in [02.1-tokenizer.md](02.1-tokenizer.md) §6.2). Binary expressions use precedence climbing with an operand stack and an operator stack, not one method per precedence level. It reports errors into a list and keeps going, so a script with errors still yields a (partial) AST. `FACT-SRC` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8191`). -In scope: entry points, the rule structure, statement and expression parsing, command parsing, operator precedence, error recovery, parse-time side effects. Out of scope: AST classes ([02.3-ast.md](02.3-ast.md), planned T23), semantic checks run after the parse ([02.4-semantic-analysis.md](02.4-semantic-analysis.md), planned T24). +In scope: entry points, the rule structure, statement and expression parsing, command parsing, operator precedence, error recovery, parse-time side effects. Out of scope: AST classes ([02.3-ast.md](02.3-ast.md)), semantic checks run after the parse ([02.4-semantic-analysis.md](02.4-semantic-analysis.md), planned T24). ## 2. Source map @@ -57,7 +57,7 @@ Any exception that is not a syntax error escaping `ParseFile`/`ParseInput` becom ## 5. Data model -- The parser produces AST nodes (catalog in [AST-CATALOG.md](AST-CATALOG.md), planned T23): `ScriptBlockAst` → `ParamBlockAst`, `NamedBlockAst`, `StatementBlockAst` → statements (`PipelineAst`, `PipelineChainAst`, `AssignmentStatementAst`, `IfStatementAst`, loops, `TryStatementAst`, `FunctionDefinitionAst`, `TypeDefinitionAst`, `UsingStatementAst`, ...) → `CommandAst` / `CommandExpressionAst` → expressions. +- The parser produces AST nodes (catalog in [AST-CATALOG.md](AST-CATALOG.md)): `ScriptBlockAst` → `ParamBlockAst`, `NamedBlockAst`, `StatementBlockAst` → statements (`PipelineAst`, `PipelineChainAst`, `AssignmentStatementAst`, `IfStatementAst`, loops, `TryStatementAst`, `FunctionDefinitionAst`, `TypeDefinitionAst`, `UsingStatementAst`, ...) → `CommandAst` / `CommandExpressionAst` → expressions. - Error nodes: `ErrorStatementAst` and `ErrorExpressionAst` hold the sub-nodes parsed before the error so tools (completion) can still use them. - Positions: every node gets an extent built by helpers `ExtentOf`, `Before`, `After`, `ExtentFromFirstOf`, `LastCharacterOf` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L572-L702`). - `$^` and `$$` are updated from the first/last token of each statement via `SetPreviousFirstLastToken` (ignores tokens whose text is `$^` or `$$`) (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetPreviousFirstLastToken@L317-L342`). @@ -341,7 +341,7 @@ Generated from `TokenFlags` (`GENERATED`, 52 b | 13 | Range | `DotDot` | `..` | | | DisallowedInRestrictedMode | -Notes (`FACT-SRC`): comparison, `-as`, `-is`, `-join` (binary form), `-split` forms and `-shl`/`-shr` all share one level (5), so `-shl` binds like `-eq`; `??` is its own level (7) above comparison and below `+`; `-f` (12) binds tighter than `*` (10), and `..` (13) tighter than `-f`; `-and`/`-or`/`-xor` share level 1 so `$a -or $b -and $c` groups left to right. The single authoritative operator table is `OPERATOR-TABLE.md` (planned) (CF-OP-001, planned T28); this section is the parser's view. +Notes (`FACT-SRC`): comparison, `-as`, `-is`, `-join` (binary form), `-split` forms and `-shl`/`-shr` all share one level (5), so `-shl` binds like `-eq`; `??` is its own level (7) above comparison and below `+`; `-f` (12) binds tighter than `*` (10), and `..` (13) tighter than `-f`; `-and`/`-or`/`-xor` share level 1 so `$a -or $b -and $c` groups left to right. The single authoritative operator table is [OPERATOR-TABLE.md](OPERATOR-TABLE.md) (CF-OP-001); this section is the parser's view. ### 7.3 Ternary, unary, array literal, postfix @@ -374,7 +374,7 @@ A `Parser` instance is single-threaded (own `Tokenizer`, own error list). The ar ## 11. Extension points and customization hooks -New statement keyword: add a `TokenKind` + keyword table row (tokenizer), a `case` in `StatementRule`, a `*Rule` method, an AST class, semantic checks. New operator: tokenizer tables + `TokenFlags` precedence; `BinaryExpressionRule` needs no change for a binary operator with an existing level. Dynamic keywords (DSC/custom) plug in through `DynamicKeyword.PreParse/PostParse/SemanticCheck`. Recipes: HOWTO-extend-language (planned T210). +New statement keyword: add a `TokenKind` + keyword table row (tokenizer), a `case` in `StatementRule`, a `*Rule` method, an AST class, semantic checks. New operator: tokenizer tables + `TokenFlags` precedence; `BinaryExpressionRule` needs no change for a binary operator with an existing level. Dynamic keywords (DSC/custom) plug in through `DynamicKeyword.PreParse/PostParse/SemanticCheck`. Recipes: [HOWTO-extend-language.md](HOWTO-extend-language.md). ## 12. Dependencies diff --git a/docs-out/02-language/02.3-ast.md b/docs-out/02-language/02.3-ast.md index 9003e72..1e2146c 100644 --- a/docs-out/02-language/02.3-ast.md +++ b/docs-out/02-language/02.3-ast.md @@ -146,7 +146,7 @@ AST objects are shared between threads after construction (script blocks are cac ## 11. Extension points and customization hooks -Subclass `AstVisitor`/`AstVisitor2` (pre-order walk with skip/stop), `DefaultCustomAstVisitor(2)` (return values), or implement `IAstPostVisitHandler` for a post-order hook. Adding a node means: a class, `Copy`, `Accept`, `InternalVisit`, a `VisitX` in both interface families (with a default body), the debug-only `CheckAllParentsSet` visitor, the compiler `VisitX`, and semantic/variable-analysis handling. Recipes: HOWTO-extend-language (planned T210). +Subclass `AstVisitor`/`AstVisitor2` (pre-order walk with skip/stop), `DefaultCustomAstVisitor(2)` (return values), or implement `IAstPostVisitHandler` for a post-order hook. Adding a node means: a class, `Copy`, `Accept`, `InternalVisit`, a `VisitX` in both interface families (with a default body), the debug-only `CheckAllParentsSet` visitor, the compiler `VisitX`, and semantic/variable-analysis handling. Recipes: [HOWTO-extend-language.md](HOWTO-extend-language.md). ## 12. Dependencies diff --git a/docs-out/02-language/02.5-compiler-and-engine.md b/docs-out/02-language/02.5-compiler-and-engine.md index 0643bfc..bf16817 100644 --- a/docs-out/02-language/02.5-compiler-and-engine.md +++ b/docs-out/02-language/02.5-compiler-and-engine.md @@ -143,7 +143,7 @@ Compilation of each variant is under `lock(this)` with double-checked flags; the ## 11. Extension points and customization hooks -Attribute generators registered in `Compiler`'s static constructor; `InternalTestHooks` (cache bypass); `ScriptBlock` language-mode capture at creation. Recipes for adding syntax: HOWTO-extend-language (planned T210). +Attribute generators registered in `Compiler`'s static constructor; `InternalTestHooks` (cache bypass); `ScriptBlock` language-mode capture at creation. Recipes for adding syntax: [HOWTO-extend-language.md](HOWTO-extend-language.md). ## 12. Dependencies diff --git a/docs-out/02-language/02.6-types-and-operators.md b/docs-out/02-language/02.6-types-and-operators.md new file mode 100644 index 0000000..4f8dcf8 --- /dev/null +++ b/docs-out/02-language/02.6-types-and-operators.md @@ -0,0 +1,138 @@ +--- +id: LANG-TYPES-001 +title: Types and operators +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-PIP-001] +sources: [src/System.Management.Automation/engine/LanguagePrimitives.cs, src/System.Management.Automation/engine/runtime/Binding/Binders.cs, src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs, src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs, src/System.Management.Automation/engine/lang/parserutils.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/members-tierA.json, data/binders.json] +--- + +# Types and operators + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Detail documents: [OPERATOR-TABLE.md](OPERATOR-TABLE.md) (CF-OP-001), [CONVERSION-RULES.md](CONVERSION-RULES.md) (CF-CNV-001). Binder mechanics: [BINDER-CATALOG.md](BINDER-CATALOG.md). + +## 1. Purpose and scope [SPEC] + +How values behave in expressions: the value model (`$null`, `AutomationNull`, `PSObject` wrappers, numeric types), arithmetic and promotion, comparison, truthiness, string handling, collection enumeration ("unrolling"), and the regex-style operators with `$Matches`. This is the semantic core a new interpreter must reproduce to give identical results. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives@L298-L5885 | static class | conversion, truthiness, equality, comparison, enumeration | A | +| src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422 | class | binary operators on arbitrary values | A | +| src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder@L518-L780 | class | "is this value enumerated, and how" | A | +| src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipeWriterBinder@L841-L926 | class | writes a statement's value to the output pipe | A | +| src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129 | static class | `int` arithmetic with overflow to `double` (siblings for uint, long, ulong, decimal, double) | A | +| src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1624 | static class | `-like`, `-match`, `-replace`, `-split`, `-join`, range, `-contains` helpers | A | + +## 3. Responsibilities and non-responsibilities + +Value semantics of operators and enumeration. Not responsible for syntax (parser), output formatting (T36), member access (ETS, T35). + +## 4. Key types and contracts [IMPL] + +- **`$null`** is a real null reference. **`AutomationNull.Value`** is a distinct "no value" object: a pipeline or statement that produced nothing evaluates to it; it enumerates as an empty sequence, writes nothing to the pipe, and converts to `$null` when stored. +- **`PSObject`** wraps any .NET object to carry extra (ETS) members; most operations call `PSObject.Base` first, and binders defer to a PSObject-aware path when an operand is a wrapper with a different base object. +- Numeric type order for arithmetic is the `TypeCode` order (`byte`/`sbyte`/`int16`/`uint16` < `int32` < `uint32` < `int64` < `uint64` < `single`/`double` < `decimal`), see [OPERATOR-TABLE.md](OPERATOR-TABLE.md) and `BinaryNumericOp`. + +## 5. Data model + +Value kinds the operators distinguish: null, string, char, bool, numeric (11 CLR types + `BigInteger`), enum, collection (array, `IEnumerable`, `IList`), dictionary (`IDictionary`/`Hashtable`), `ScriptBlock`, `PSObject`, other objects (with optional `op_*` operators). + +## 6. Control flow + +Operator evaluation: dynamic binding site → rule chosen from the runtime types of the operands (cached per site) → helper call. Rules are described per operator in [OPERATOR-TABLE.md](OPERATOR-TABLE.md). + +## 7. Algorithms + +### 7.1 Enumeration and output unrolling (CF-PIP-001) + +**Output unrolling and enumeration rules.** A value is *enumerated* (treated as a sequence) when, after unwrapping `PSObject`: it is `AutomationNull` (empty sequence), an array, a `DataTable` (its rows), a COM enumerable, any other `IEnumerable` that is not a string, `IDictionary` or `XmlNode` (the generic `IEnumerable` enumerator is preferred), or an `IEnumerator`; `null`, strings, dictionaries (hashtables), `XmlNode` and all other objects are **not** enumerated and are single values. When a statement writes its value to the output pipe: `AutomationNull` writes nothing; a non-enumerable value is written as one object (mutable value types are copied first); an enumerable value is **unrolled exactly one level**: each element is written separately (elements that are themselves collections stay whole), and an enumerator is disposed afterwards if the engine created it. + +Evidence: src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L693, src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipeWriterBinder@L841-L926, src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::EnumerableOps.WriteEnumerableToPipe@L3623-L3640, src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.CalculateGetEnumerable@L527-L543, [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.5 (capture), src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::PipelineOps@L29-L889. Capturing output in `$x = …`, `@(…)` or `$(…)` collects what was written to a temporary pipe (`PipelineOps.PipelineResult`); the usual result shapes (no output → `$null`, one object → that object, several → `object[]`; `@(…)` always an array) were **not** read in `PipelineResult` and are `UNVERIFIED` (VB-019). + +### 7.2 Arithmetic and promotion + +For numeric operands the wider `TypeCode` selects the implementation class (`IntOps`, `UIntOps`, `LongOps`, `ULongOps`, `DecimalOps`, `DoubleOps`); `int` addition, subtraction and multiplication that overflow return a `double`; `int` division returns an `int` if exact and a `double` otherwise; division or remainder by zero raises a `RuntimeException`; a signed negative operand mixed with `uint`/`ulong` falls to `long`/`decimal` (with double fallback on overflow). A string operand of a numeric operator is converted with the language number scanner (CF-LEX-002). Evidence: src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129, src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.BinaryNumericOp@L2384-L2539. + +### 7.3 Comparison + +Detailed in [OPERATOR-TABLE.md](OPERATOR-TABLE.md) and [CONVERSION-RULES.md](CONVERSION-RULES.md) §7.6: left operand drives; strings compare with the culture comparer (plain and `i` operators ignore case, `c` operators do not); a collection on the left makes comparison operators **filter** instead of returning a bool; `$null` rules (`$null -eq x` only for `$null`; `x -eq $null` false for non-null scalars). + +### 7.4 Regex and wildcard operators + +`-match`/`-notmatch`: regex (culture-aware options by case flag); scalar left operand sets `$Matches` (hashtable with case-insensitive keys, numeric group names as integer keys); array left operand returns the matching elements. `-like`/`-notlike`: wildcard pattern; array left operand filters. `-replace`: regex replace with a replacement string or a script block evaluator. `-split`/`-join`: described in the table. Regex objects are cached by (options, pattern) with a cap of 1,000 per option set that clears the set when exceeded. Evidence: src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1624 (notes W2-W5). + +### 7.5 Strings + +String `+` appends the right operand's string form; `*` repeats; indexing, slicing and `-f` follow the table; string comparison uses the culture of the operation (invariant for conversions and the operators unless a culture is passed). Double-quoted expansion builds the string with `-f`-style formatting of nested expressions each converted with the `PSToStringBinder` (enumerables joined with `$OFS`, default space). + +## 8. Observable behavior [SPEC] + +Read from source: `2147483647 + 1` gives a `double`; `7 / 2` gives `3.5` and `6 / 2` gives `3` (an `int`); `"a" * 3` is `"aaa"`; `@(1,2) + 3` is a 3-element array; `1,2 -eq 2` returns `2` (filter) not `$true`; `$null -eq $null` is `$true`; `"abc" -like "a*"` is `$true`; `[bool]@(0)` is `$false`. No golden sample: STATIC-ONLY. Tests: `test/powershell/Language/Operators/` (`data/tests.json`; mapping `UNVERIFIED`). + +## 9. Error handling and edge cases + +`PSInvalidCastException` for failed conversions; `RuntimeException` wrapping `DivideByZeroException`; "cannot apply operator" errors (`op_*` not found) from `CallImplicitOp`; `AddHashTableToNonHashTable`; regex errors wrapped as `InvalidRegularExpression`; comparison failures as `PSArgumentException`. + +## 10. Concurrency and threading + +Binder rules and conversion caches are shared across threads (concurrent dictionaries/locks); `$Matches` is a scope variable (per runspace). + +## 11. Extension points and customization hooks + +User types can define `op_Addition`, `op_Subtraction`, `op_Multiply`, `op_Division`, `op_Modulus` (looked up via `CallImplicitOp`, preferring the left operand's type unless it is primitive or null), `IComparable`, `Parse`, and type converters. + +## 12. Dependencies + +Inbound: compiled code (binders), parameter binding, `Where-Object`/`Sort-Object` (comparison). Outbound: ETS (`PSObject`), conversion layer, regex engine, culture info. + +## 13. Performance notes + +Binder rules are cached per call site by operand types; integer arithmetic avoids boxing in typed fast paths (`IntOps` returns boxed objects only because results may switch type). + +## 14. Security considerations (mandatory) + +User-defined operator and conversion methods run code during an innocent-looking expression; Constrained Language Mode restricts which types participate (conversion gate, binder language checks). Regex cache and `-split`/`-replace` with attacker-controlled patterns can cause catastrophic backtracking (no timeout was found in `NewRegex`, `UNVERIFIED`). + +## 15. Enterprise considerations + +`string * n` and array repetition are capped at 1,024 only in RestrictedLanguage mode (data sections); a negative string repeat count throws `ArgumentOutOfRangeException`; no cap outside data sections was found; `Range` allocates the full array. + +## 16. Tests + +Not read per assertion (VB-005). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| left-operand-driven typing | `FACT-SRC` | asymmetric results (`1+"2"` vs `"1"+2`); easy to implement, surprising | +| arithmetic overflow promotes to double | `FACT-SRC` | no integer overflow exceptions in scripts | +| comparison operators filter collections | `FACT-SRC` | concise filtering, but `if ($a -eq 1)` with arrays is truthy on non-empty result | +| output unrolls one level | `FACT-SRC` | pipelines stream elements; nested arrays flatten one level only | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-TYPES-01** [REPLICATE] (area: NLD-TYP) Implement the enumeration rule and one-level unrolling exactly as in §7.1, including `AutomationNull`, strings and dictionaries as non-enumerated. Evidence: src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L693. +- **LSN-TYPES-02** [REPLICATE] (area: NLD-OP) Arithmetic: wider-type promotion, `int` overflow to `double`, exact `int/int` stays `int`, zero division errors. Evidence: src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129. +- **LSN-TYPES-03** [REPLICATE] (area: NLD-OP) Collections on the left of comparison, `-like` and `-match` filter; scalar `-match` sets `$Matches`. Evidence: src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.BinaryComparisonCommon@L3378-L3419. +- **LSN-TYPES-04** [ADAPT] (area: NLD-OP) Implement `op_*` user-defined operators only for MicroteamShell's own types. Why: avoids reflection over arbitrary .NET types; same observable behavior for supported types. Evidence: src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.CallImplicitOp@L2290-L2311. +- **LSN-TYPES-05** [ADAPT] (area: NLD-TYP) Add a regex match timeout and size limits for string/array repetition in every mode. Why: PowerShell caps repetition only in restricted mode and no regex timeout was found (`UNVERIFIED`); a personal shell should fail safely. Evidence: src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1624. + +## 19. Open questions / UNVERIFIED + +- `PipelineOps.PipelineResult` (how a captured list becomes `$null`/object/array) was not read (VB-019). +- `PSUnaryOperationBinder`, shifts, bitwise operators, `-split` details, `-f` semantic and `PSObject` comparison inside binders were read at dispatch level only. +- `$Matches` on array left operands. +- Regex timeout and repetition limits. + +Verification: `PSEnumerableBinder.FallbackConvert`, `PSPipeWriterBinder`, `EnumerableOps.WriteEnumerableToPipe`, `LanguagePrimitives` enumeration (L449-L560), `NumericOps.IntOps` (L15-L130), `BinaryNumericOp` (L2384-L2470), `BinaryAdd`/`BinaryMultiply`/`BinarySubDivOrRem`/`CompareEQ`/`BinaryEqualityComparison`/`BinaryComparisonCommon` were read; `ParserOps` from the earlier notes (`parserutils.cs` W1-W7). diff --git a/docs-out/02-language/02.7-language-features.md b/docs-out/02-language/02.7-language-features.md new file mode 100644 index 0000000..315d898 --- /dev/null +++ b/docs-out/02-language/02.7-language-features.md @@ -0,0 +1,142 @@ +--- +id: LANG-FEATURES-001 +title: Language features +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Compiler.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [] +--- + +# Language features + +> STATIC-ONLY — behavior not run-verified. A feature map: for each language feature, its scope class, the stage that implements it and where this study describes it. The map is the input to the compatibility matrix of the synthesis phase. Scope classes follow the study (CORE / EXTENDED / OUT-OF-SCOPE-DOCUMENTED). + +## 1. Purpose and scope [SPEC] + +Find any language feature in the documents quickly and see how it travels through tokenizer, parser, AST, checks, compiler and runtime. Evidence is by link to the owning document section; each document cites the source. + +## 2. Source map + +Stages and documents: tokenizer [02.1](02.1-tokenizer.md) · parser [02.2](02.2-parser.md) · grammar [GRAMMAR](GRAMMAR.ebnf.md) · AST [02.3](02.3-ast.md) · checks [02.4](02.4-semantic-analysis.md) · compile/run [02.5](02.5-compiler-and-engine.md) · types/operators [02.6](02.6-types-and-operators.md) · scopes [SCOPE](SCOPE-AND-VARIABLES.md) · script blocks [SB](SCRIPT-BLOCKS.md) · runtime helpers [RTOPS](RUNTIME-OPERATIONS.md). + +## 3. Responsibilities and non-responsibilities + +Navigation only; behavior statements live in the linked sections. + +## 4. Key types and contracts [IMPL] + +Each row names the main AST node(s) and the `Compiler` method (full lists: [AST-CATALOG.md](AST-CATALOG.md), [COMPILATION-MAP.md](COMPILATION-MAP.md)). + +## 5. Data model — feature table + +| Feature | Class | Tokenizer / parser | AST | Executed by | Described in | +|---|---|---|---|---|---| +| Command mode vs expression mode | CORE | mode switching ([02.1 §6.2](02.1-tokenizer.md)) | — | parser | CF-LEX-001 | +| Comments, `<# #>`, `#requires`, line continuation | CORE | tokenizer ([02.1 §7.4](02.1-tokenizer.md)) | — | tokenizer | 02.1 | +| Numbers (hex, binary, suffixes, `kb`…) | CORE | `ScanNumberHelper` | `ConstantExpressionAst` | constant | CF-LEX-002 | +| Single/double-quoted strings, escapes | CORE | `ScanStringLiteral`/`ScanStringExpandable` | `StringConstantExpressionAst`, `ExpandableStringExpressionAst` | `StringOps.FormatOperator` | [02.1 §7.2](02.1-tokenizer.md) | +| Here-strings | CORE | `ScanHereString*` | same | same | 02.1 | +| Variables, scopes, `$global:`, `${name}` | CORE | `ScanVariable` | `VariableExpressionAst` | `VariableOps` | [SCOPE](SCOPE-AND-VARIABLES.md) CF-SCP-001 | +| Automatic and preference variables | CORE | — | `VariableExpressionAst` | tuple slots | SCOPE §5 | +| `$using:` | CORE | variable token | `UsingExpressionAst` | `VariableOps.GetUsingValue` | [SB](SCRIPT-BLOCKS.md) | +| Splatting `@args` | CORE | `SplattedVariable` token | `VariableExpressionAst` (splatted) | `PipelineOps.Splat` (not read) | [COMPILATION-MAP](COMPILATION-MAP.md) `VisitCommand` | +| Arithmetic, comparison, logical, bitwise operators | CORE | dash operators (expression mode) | `BinaryExpressionAst`, `UnaryExpressionAst` | binders | [OPERATOR-TABLE](OPERATOR-TABLE.md) CF-OP-001 | +| `-like` `-match` `-replace` `-split` `-join` `-contains` `-in` `-is` `-as` `-f` | CORE | same | same | `ParserOps`, `TypeOps` | OPERATOR-TABLE, [RTOPS](RUNTIME-OPERATIONS.md) | +| Range `..` | CORE | `DotDot` | `BinaryExpressionAst` | `ParserOps.RangeOperator` | OPERATOR-TABLE | +| Ternary `?:`, null-coalescing `??`, `??=`, `?.`, `?[]` | EXTENDED | expression-mode tokens | `TernaryExpressionAst`, member/index nodes with `NullConditional` | compiler | [02.2 §7.3](02.2-parser.md) | +| Arrays, comma operator, `@()` | CORE | `ArrayLiteralRule` | `ArrayLiteralAst`, `ArrayExpressionAst` | `object[]` | [02.6 §7.1](02.6-types-and-operators.md) | +| Hashtables `@{}` (values are statements) | CORE | `HashExpressionRule` | `HashtableAst` | `HashtableOps` | GRAMMAR §7 | +| Sub-expressions `$()`, parentheses | CORE | `SubExpressionRule` | `SubExpressionAst`, `ParenExpressionAst` | captured pipe | [02.5 §6.5](02.5-compiler-and-engine.md) | +| Type literals, casts, `[type]::member` | CORE | `TypeNameRule` | `TypeExpressionAst`, `ConvertExpressionAst`, `MemberExpressionAst` | conversions, binders | [CONVERSION](CONVERSION-RULES.md) | +| Member access `.`, `::`, method calls, `.Where{}` | CORE | `GetMemberAccessOperator` | `MemberExpressionAst`, `InvokeMemberExpressionAst` | member binders | [BINDER-CATALOG](BINDER-CATALOG.md) | +| Indexing, slicing, `?[]` | CORE | `ElementAccessRule` | `IndexExpressionAst` | index binders | BINDER-CATALOG | +| Assignment, `+=`, destructuring | CORE | `PipelineChainRule` | `AssignmentStatementAst` | `CompileAssignment` | [02.5 §7.3](02.5-compiler-and-engine.md) | +| Commands, arguments, parameters, `--%` | CORE | `CommandRule` | `CommandAst`, `CommandParameterAst` | command processors (T32/T33) | [02.2 §7.4](02.2-parser.md) | +| Pipelines, `&&`, `||`, background `&` | CORE (chains EXTENDED) | `PipelineRule`, `PipelineChainRule` | `PipelineAst`, `PipelineChainAst` | `PipelineOps` | 02.2 §6.3 | +| Redirection `>`, `>>`, `2>&1`, `*>` | CORE | redirection tokens | `FileRedirectionAst`, `MergingRedirectionAst` | `FileRedirection` | [02.1 §7.5](02.1-tokenizer.md) | +| `if`/`elseif`/`else` | CORE | `IfStatementRule` | `IfStatementAst` | `IfThenElse` | COMPILATION-MAP | +| `while`, `do`, `for`, `foreach`, labels | CORE | loop rules | loop AST nodes | `GenerateWhileLoop` | [02.5 §6.3](02.5-compiler-and-engine.md) | +| `switch` (flags, `-file`) | CORE | `SwitchStatementRule` | `SwitchStatementAst` | `VisitSwitchStatement` | 02.5 §6.3 | +| `break`, `continue`, `return`, `exit`, `throw` | CORE | flow rules | flow AST nodes | goto or exceptions | 02.5 §6.4 | +| `try`/`catch`/`finally`, `trap` | CORE | `TryStatementRule` | `TryStatementAst`, `TrapStatementAst` | `VisitTryStatement`, `ProcessTraps` | 02.5 §6.4, [RTOPS §6.1](RUNTIME-OPERATIONS.md) | +| Functions, filters, `param()`, `[CmdletBinding()]`, `begin/process/end/clean` | CORE | `FunctionDeclarationRule`, `ParamBlockRule` | `FunctionDefinitionAst`, `ParamBlockAst`, `NamedBlockAst` | lazy compile | 02.5, [SB](SCRIPT-BLOCKS.md) | +| Script blocks, closures | CORE | `ScriptBlockExpressionRule` | `ScriptBlockExpressionAst` | `ScriptBlock` | SB | +| Dot-sourcing `.`, call `&` | CORE | `CommandRule` invocation operator | `CommandAst` (invocation operator) | `InvokeWithPipeImpl` | SCOPE §7.1 | +| `data` sections | EXTENDED | `DataStatementRule` | `DataStatementAst` | restricted language | [02.4 §14](02.4-semantic-analysis.md) | +| `using namespace/module/assembly/type` | EXTENDED | `UsingStatementRule` | `UsingStatementAst` | symbol resolution, usings prologue | 02.4 | +| Classes and enums | EXTENDED | `ClassDefinitionRule`, `EnumDefinitionRule` | `TypeDefinitionAst` and members | `TypeDefiner` | 02.4 §6 | +| Modules, manifests, `Import-Module` | EXTENDED | — | — | module system (T37) | planned | +| Providers and drives (`env:`, `variable:`) | EXTENDED | drive-qualified names | — | session state | SCOPE §4 | +| `configuration`, dynamic keywords (DSC) | OUT-OF-SCOPE-DOCUMENTED | `ConfigurationStatementRule`, `DynamicKeywordStatementRule` | `ConfigurationDefinitionAst`, `DynamicKeywordStatementAst` | DSC | 02.2 §14 | +| `workflow`, `parallel`, `sequence`, `inlinescript` | OUT-OF-SCOPE-DOCUMENTED | keywords, reserved | `BlockStatementAst` | not supported in PowerShell 7 (`WorkflowNotSupportedInPowerShellCore`) | 02.4 table | +| `from`, `define`, `var` | reserved | keywords | `ErrorStatementAst` | `ReservedKeywordNotAllowed` | [GRAMMAR §4](GRAMMAR.ebnf.md) | + +Rows: 39 (hand-maintained; counted by `tools/gen_T210.py --count`). + +## 6. Control flow + +Not applicable (index document). + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +See the linked sections; no golden samples (STATIC-ONLY). + +## 9. Error handling and edge cases + +Each feature's errors are in [ERROR-CATALOG.md](ERROR-CATALOG.md) (parse) and [RTOPS](RUNTIME-OPERATIONS.md) (run time). + +## 10. Concurrency and threading + +Not applicable. + +## 11. Extension points and customization hooks + +[HOWTO-extend-language.md](HOWTO-extend-language.md). + +## 12. Dependencies + +All 02-language documents. + +## 13. Performance notes + +Not applicable. + +## 14. Security considerations (mandatory) + +Features with a security dimension: `data` sections and restricted language (02.4 §14), `using assembly`/`using module` at parse time (02.2 §14), class and configuration gates in Constrained Language Mode (02.2 §14), `$using:` and script block language mode (SB §14), type conversion gate ([CONVERSION §14](CONVERSION-RULES.md)). + +## 15. Enterprise considerations + +Not applicable. + +## 16. Tests + +Not applicable. + +## 17. Design decisions and trade-offs + +The map shows the language is mostly expression-and-statement CORE with a large EXTENDED rim (classes, modules, DSC, providers) that a personal shell can phase in. + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-FEAT-01** [REPLICATE] (area: NLD-GRM) Build the CORE rows first, in the order tokenizer, parser, AST, evaluator, scopes, operators, commands/pipeline, flow control, errors. Evidence: src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8187. +- **LSN-FEAT-02** [ADAPT] (area: NLD-GRM) Add the EXTENDED rows (classes, modules, providers) only after the CORE rows pass the corpus; treat DSC and workflow as absent. Why: same language core with far less surface. Evidence: src/System.Management.Automation/engine/parser/Parser.cs::Parser.ConfigurationStatementRule@L2893-L3322. +- **LSN-FEAT-03** [AVOID] (area: NLD-GRM) Do not implement `workflow`/`parallel`/`sequence`/`inlinescript` or the reserved keywords `from`, `define`, `var`; reject them with the same kind of parse error. Evidence: src/System.Management.Automation/engine/parser/Parser.cs::Parser.BlockStatementRule@L2341-L2359. DEV: DEV-FEAT-091 + +## 19. Open questions / UNVERIFIED + +- Splatting runtime behavior (`PipelineOps.Splat`) and `.Where{}`/`.ForEach{}` intrinsic methods were not read (VB-021). +- Module and provider features are described only as pointers. + +Verification: each row was built from a section of an existing document; every referenced section exists (checked by `check-links`). diff --git a/docs-out/02-language/02.8-language-tooling.md b/docs-out/02-language/02.8-language-tooling.md new file mode 100644 index 0000000..755d3b0 --- /dev/null +++ b/docs-out/02-language/02.8-language-tooling.md @@ -0,0 +1,128 @@ +--- +id: LANG-TOOLING-001 +title: Language tooling +kind: subsystem +tier: B +status: draft +run_verification: none +scope_class: EXTENDED +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/lang/interface/PSParser.cs, src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs, src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/ast.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/CommandCompletion/_folder.md] +generated_from: [data/visitors.json] +--- + +# Language tooling + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope `EXTENDED`: the engine exposes its tokenizer, parser, AST, type inference and completion to editors and analyzers. MicroteamShell does not need these for the language core, but its design should keep the same hooks possible. Completion internals (≈15,000 lines) are Tier C and described only by API surface. + +## 1. Purpose and scope [SPEC] + +What a tool (editor, linter, formatter, test harness) can obtain from the engine without running a script: tokens, comments, an AST with extents and parse errors, constant evaluation of data literals, help comments, script requirements, inferred types, and completion results. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8187 | public sealed class | `ParseInput`, `ParseFile` return AST, token array, error array ([02.2-parser.md](02.2-parser.md) §4) | A | +| src/System.Management.Automation/engine/lang/interface/PSParser.cs::PSParser@L64-L193 | public sealed class | legacy tokenizer API `Tokenize(string, out errors)` | B | +| src/System.Management.Automation/engine/lang/interface/PSToken.cs::PSToken@L28-L363 | public sealed class | legacy token with `PSTokenType` | B | +| src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs::AstTypeInference@L40-L110 | internal static class | type inference for completion (public API is a wrapper elsewhere, hint mismatch) | A | +| src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs::CommandCompletion@L20-L563 | public class | `CompleteInput`, `MapStringInputToParsedInput`, `GetNextResult` | C | + +## 3. Responsibilities and non-responsibilities + +Expose analysis results; no execution. Execution-dependent helpers (completion of command names, parameter values, type inference through commands) call into the runtime and **can run code**, see §14. + +## 4. Key types and contracts [IMPL] + +- **Parsing for tools**: `Parser.ParseInput(input, out Token[] tokens, out ParseError[] errors)` and `ParseFile(path, …)` return the root `ScriptBlockAst`; tokens include comments, newlines and line continuations because the token list is requested ([02.1-tokenizer.md](02.1-tokenizer.md) §4: `TokenList`); errors are the full list of CF-ERR-002; the AST has error nodes for broken input ([02.3-ast.md](02.3-ast.md) §4). +- **Legacy tokens**: `PSParser.Tokenize` creates a `Parser` with `ProduceV2Tokens = true`, parses in `ParseMode.Default` with a token list, ignores any exception, drops the last token (end of input) and maps each `Token` to a `PSToken`; errors are mapped to `PSParseError`. With `ProduceV2Tokens` the parser **rewrites the saved token list** for attributes and type constraints: the tokens `[`, type name, `(` or `]` are replaced by a single `Identifier` token flagged `TypeName` (`Tokenizer.ReplaceSavedTokens`), reproducing the PowerShell 2 tokenizer's output (src/System.Management.Automation/engine/parser/Parser.cs::Parser.AttributeRule@L1054-L1153, src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ReplaceSavedTokens@L1102-L1122). +- **AST queries**: `Ast.Find`/`FindAll` with a predicate and a nested-script-block flag; `Ast.SafeGetValue()` for literal data; `ScriptBlockAst.GetHelpContent()` / `FunctionDefinitionAst.GetHelpContent()` return a `CommentHelpInfo` parsed from the comment tokens; `ScriptBlockAst.ScriptRequirements` lists `#requires` data ([02.1-tokenizer.md](02.1-tokenizer.md) §7.4). +- **Type inference** (`AstTypeInference.InferTypeOf` overloads taking an AST, optional `PowerShell` and `TypeInferenceRuntimePermissions {None, AllowSafeEval}`) returns the distinct inferred type names (distinct by name) (src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs::TypeInferenceVisitor@L550-L3285). +- **Completion**: `CommandCompletion.CompleteInput(input, cursorIndex, options[, PowerShell])` and the overload taking an existing AST, token array and cursor position; `MapStringInputToParsedInput` parses a string plus cursor into (AST, tokens, position); the result object offers `GetNextResult(forward)` for cycling (src/System.Management.Automation/engine/CommandCompletion/CommandCompletion.cs::CommandCompletion.CompleteInput@L92-L108). Extensible completers: `ExtensibleCompletion.cs` (argument completers, `CustomArgumentCompleters` on the context). + +## 5. Data model + +Token list (with comments), `ParseError[]`, AST with extents (line/column/offset), `CompletionResult` items (text, list text, type, tooltip), inferred `PSTypeName`s (synthetic type names carry member names for hashtable and custom-object shapes). + +## 6. Control flow + +```mermaid +flowchart LR + T[source text] --> P[Parser.ParseInput] + P --> A[AST + tokens + errors] + A --> Q[Find / FindAll / SafeGetValue] + A --> H[GetHelpContent] + A --> TI[AstTypeInference] + A --> C[CommandCompletion.CompleteInput] + TI --> C +``` + +## 7. Algorithms + +**Type inference** (read in the T24 notes, `TypeInferenceVisitor.cs`, `FACT-SRC`): a visitor over the AST returns possible type names per node: literals and casts give their type; arrays give element types; hashtables give a synthetic type with the key names; binary operators give `bool` for comparison/logic, `string` for replace/format/join, `string[]` for split, otherwise the left operand's types refined by public static `op_*` methods on the left CLR types; variables use the last textual assignment in the enclosing scope (a type constraint wins) with special handling for `$_`/`$PSItem` (catch, trap, switch, `Where` target, pipeline predecessor) and `$this`; commands use `OutputType` metadata, special cases for `New-Object`, `Where-Object`, `ForEach-Object`, `Select-Object`, `Group-Object`, `Get-Random`, `Get-CimInstance`, and for script functions without `OutputType` the body is analyzed with a recursion guard; members are looked up through the type table and reflection. + +**Completion** analysis (positions → context → completers) is in `CompletionAnalysis.cs` (2,693 lines) and `CompletionCompleters.cs` (9,251 lines): `UNVERIFIED`, not read (VB-023). + +## 8. Observable behavior [SPEC] + +Examples from the API shape: `[Parser]::ParseInput("1 +", …)` returns an AST plus an incomplete-input error; `PSParser.Tokenize` returns tokens without the final end-of-input token; completion returns a `CommandCompletion` with `ReplacementIndex`/`ReplacementLength`/`CompletionMatches` (names from the type; `UNVERIFIED`). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`PSParser.Tokenize` swallows exceptions from parsing, so a failure yields an empty token list and the errors collection (possibly null → `NullReferenceException` risk when `_errors` was never assigned, `INFERENCE`). Type inference requires a non-null result from the visitor and swallows `RuntimeException` when instantiating attributes. + +## 10. Concurrency and threading + +`Parser` instances are single-threaded; the dynamic-keyword table is thread-local; type inference with a `PowerShell` object needs a free runspace thread. Static caches are concurrent. + +## 11. Extension points and customization hooks + +Custom argument completers (`Register-ArgumentCompleter` equivalents), `ValidateSet` generators, `IArgumentCompleter`, class-based completers (`ExtensibleCompletion.cs`), `DynamicKeyword.SemanticCheck`. + +## 12. Dependencies + +Inbound: editors (PowerShell extension, PSReadLine in a separate repository), analyzers, tests (`PSTests.Parallel.*` AST visitor tests). Outbound: parser, AST, runtime (type table, command discovery, providers), `PowerShell` API. + +## 13. Performance notes + +Completion parses and walks the AST per request; type inference caches nothing across calls except the binder/type-table caches (`INFERENCE`). + +## 14. Security considerations (mandatory) + +Tooling is not side-effect free: type inference may call `Get-CimClass` through a `PowerShell` helper for CIM type names, may evaluate expressions with `SafeExprEvaluator` (hashtable keys and values are evaluated even without `AllowSafeEval`, an inconsistency candidate), and may resolve commands (`CommandDiscovery.LookupCommandInfo`); completion of file paths and commands touches the file system and module cache. Parsing a file for tools triggers the parse-time effects of [02.2-parser.md](02.2-parser.md) §14 (`using module`, `using assembly`) unless `ParseMode.ModuleAnalysis` is chosen. + +## 15. Enterprise considerations + +Stable public parse API and token/AST shapes are the main integration points for third-party analyzers; ETW events around parse/semantic/compile give timing data. + +## 16. Tests + +`test/xUnit/csharp/test_AstDefaultVisit.cs` and the `PSTests.Parallel` visitors (default-visit contracts); completion tests under `test/powershell/engine/Api/` and `Host/TabCompletion/` (`data/tests.json`; per-assertion mapping `UNVERIFIED`). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| tools use the same parser as the engine | `FACT-SRC` | exact agreement with execution; parse-time side effects leak to tools | +| tokens returned with comments, error nodes in the AST | `FACT-SRC` | formatters and completion work on broken code | +| legacy tokenizer API kept by rewriting tokens | `FACT-SRC` | compatibility cost inside the parser (`ProduceV2Tokens` branches) | +| type inference may execute helper commands | `FACT-SRC` | richer results, side effects | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-TOOLS-01** [REPLICATE] (area: NLD-TOOL) Expose `Parse(text)` returning AST, tokens (with comments) and all errors, with error nodes for broken input, from day one. Evidence: src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseInput@L131-L134. +- **LSN-TOOLS-02** [REPLICATE] (area: NLD-TOOL) Provide `Find/FindAll` over the AST and a pure "literal data" evaluator for manifests and configuration. Evidence: src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420. +- **LSN-TOOLS-03** [AVOID] (area: NLD-TOOL) Do not build the PowerShell 2 compatible token API or the V2 token rewriting. Evidence: src/System.Management.Automation/engine/lang/interface/PSParser.cs::PSParser@L64-L193. DEV: DEV-TOOLS-100 +- **LSN-TOOLS-04** [ADAPT] (area: NLD-TOOL) If completion is added, keep type inference side-effect free (no command execution, no CIM lookups). Why: same completion quality for the shell's own types without running code. Evidence: src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs::AstTypeInference@L40-L110. + +## 19. Open questions / UNVERIFIED + +- Completion analysis and completers (Tier C, ≈12,000 lines) not read (VB-023). +- Public wrappers around `AstTypeInference` (the internal class here is reached through other types). +- `PSTokenType` mapping table (`GetPSTokenType`) not read. + +Verification: `PSParser.cs` (194 lines) read in full; `Parser.AttributeRule` V2 branch (L1136-L1141) and `ParseInput`/`ParseFile` re-read; `TypeInferenceVisitor.cs` from the T24 notes (3,338 lines, full); `CommandCompletion.cs` public signatures only. diff --git a/docs-out/02-language/BINDER-CATALOG.md b/docs-out/02-language/BINDER-CATALOG.md new file mode 100644 index 0000000..53a11b6 --- /dev/null +++ b/docs-out/02-language/BINDER-CATALOG.md @@ -0,0 +1,83 @@ +--- +id: LANG-BINDERCAT-001 +title: Binder catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/runtime/Binding/Binders.cs, src/System.Management.Automation/engine/ComInterop/ComBinder.cs, src/System.Management.Automation/engine/ComInterop/ComInvokeAction.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/runtime/_folder.md] +generated_from: [data/binders.json] +--- + +# Binder catalog + +> STATIC-ONLY — behavior not run-verified. The table is `GENERATED` by `docs-out/tools/gen_T26.py` from `data/binders.json` (`APPROX`) joined with hand annotations (`docs-out/tools/annotations/T26-binders.json`). Annotation level **read** = the binder's bind/fallback body was read (`FACT-SRC`); **names** = only member names and the compiler's use were read, details `UNVERIFIED` (VB-020). Narrative: [02.6-types-and-operators.md](02.6-types-and-operators.md), [RUNTIME-OPERATIONS.md](RUNTIME-OPERATIONS.md), [IR-SPEC.md](IR-SPEC.md) §5. + +## 1. Purpose and scope [SPEC] + +A *binder* is a DLR object that turns an operation on runtime values into code, and caches the result per call site by operand types. Every non-trivial operation in compiled PowerShell code (operators, casts, member access, indexing, enumeration, output writes) is a dynamic call site with one of these binders. The scanner found 30 binder classes (26 in `Binders.cs`, the rest COM interop), with 164 creation or lookup sites. + +## 2. Source map + +| path | role | +|---|---| +| `src/System.Management.Automation/engine/runtime/Binding/Binders.cs` | all PowerShell binders (8,015 lines) | +| `src/System.Management.Automation/engine/ComInterop/ComBinder.cs`, `ComInvokeAction.cs` | COM member and invoke binders (out of scope, listed for completeness) | + +## 3. Binder table + +| Binder | Base | Operation (annotation) | Level | Used for | Creation sites | Source | +|---|---|---|---|---|---:|---| +| `ComBinder.ComGetMemberBinder` | `GetMemberBinder` | COM interop binder (out of scope). | names | COM objects | 1 | `src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComGetMemberBinder@L308-L336` | +| `ComBinder.ComInvokeMemberBinder` | `InvokeMemberBinder` | COM interop binder (out of scope). | names | COM objects | 1 | `src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComInvokeMemberBinder@L341-L375` | +| `ComInvokeAction` | `InvokeBinder` | COM interop binder (out of scope). | names | COM objects | 1 | `src/System.Management.Automation/engine/ComInterop/ComInvokeAction.cs::ComInvokeAction@L16-L41` | +| `SplatInvokeBinder` | `CallSiteBinder` | COM interop binder (out of scope). | names | COM objects | 1 | `src/System.Management.Automation/engine/ComInterop/ComInvokeAction.cs::SplatInvokeBinder@L47-L83` | +| `PSEnumerableBinder` | `ConvertBinder` | Convert a value to an IEnumerator following the enumeration rule (CF-PIP-001): AutomationNull = empty; null, string, PSObject-without-base, IDictionary, XmlNode = not enumerable (null result); arrays, DataTable rows, COM enumerables, IEnumerable (typed enumerator preferred), IEnumerable, IEnumerator = enumerated. | read | foreach, switch, pipelines, @(), operators that accept collections | 8 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder@L518-L780` | +| `PSToObjectArrayBinder` | `DynamicMetaObjectBinder` | Value to object[]: AutomationNull = empty array; non-enumerable = one-element array; List = ToArray; any enumerable = array of its elements. | read | `@( … )` over a single value | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSToObjectArrayBinder@L785-L839` | +| `PSPipeWriterBinder` | `DynamicMetaObjectBinder` | Write a statement value to the output pipe: AutomationNull writes nothing; non-enumerable written as one object (mutable value types copied via PSVariableAssignmentBinder); enumerable unrolled one level and disposed. | read | every bare expression statement (CallAddPipe) | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipeWriterBinder@L841-L926` | +| `PSArrayAssignmentRHSBinder` | `DynamicMetaObjectBinder` | Shape the right side of `$a,$b,$c = …` to exactly n elements: list of n = as is; fewer = padded with $null; more = last target receives the rest as a slice; non-list = first target gets it, others $null. | read | array destructuring assignment | 3 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSArrayAssignmentRHSBinder@L935-L1042` | +| `PSToStringBinder` | `DynamicMetaObjectBinder` | Convert a value to string for expandable strings, -f arguments and file names (ToString via the engine string parser; InvokeToString helper). | names | expandable strings, format operator, redirection file names | 16 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSToStringBinder@L1054-L1114` | +| `PSPipelineResultToBoolBinder` | `DynamicMetaObjectBinder` | Captured pipeline list to bool: 0 elements = false; 1 element = that element converted to bool; more than 1 = true. | read | conditions of if/while/for/switch/do | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipelineResultToBoolBinder@L1119-L1178` | +| `PSInvokeDynamicMemberBinder` | `DynamicMetaObjectBinder` | Invoke a method whose name is computed at run time (`$o.$name(args)`). | names | dynamic member invocation | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSInvokeDynamicMemberBinder@L1180-L1276` | +| `PSGetDynamicMemberBinder` | `DynamicMetaObjectBinder` | Get a member whose name is computed at run time (`$o.$name`). | names | dynamic member read | 3 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSGetDynamicMemberBinder@L1291-L1399` | +| `PSSetDynamicMemberBinder` | `DynamicMetaObjectBinder` | Set a member whose name is computed at run time. | names | dynamic member write | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSSetDynamicMemberBinder@L1401-L1491` | +| `PSSwitchClauseEvalBinder` | `DynamicMetaObjectBinder` | Evaluate a non-constant switch clause expression against `$_` according to the switch flags (regex, wildcard, exact, case-sensitive, script block). | names | switch clauses that are not constants | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSSwitchClauseEvalBinder@L1493-L1609` | +| `PSAttributeGenerator` | `CreateInstanceBinder` | Create an attribute instance from positional and named argument values (CreateInstanceBinder); compiled lambda cached per call shape. | names | attribute creation for parameters/functions | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSAttributeGenerator@L1613-L1792` | +| `PSCustomObjectConverter` | `DynamicMetaObjectBinder` | Convert an ordered dictionary to a PSCustomObject. | names | `[pscustomobject]@{…}` | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSCustomObjectConverter@L1794-L1828` | +| `PSDynamicConvertBinder` | `DynamicMetaObjectBinder` | Conversion whose target type is resolved at run time (type name unresolved at compile time). | names | `[SomeType]$x` with late-bound type | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSDynamicConvertBinder@L1830-L1863` | +| `PSVariableAssignmentBinder` | `DynamicMetaObjectBinder` | Copy mutable value types when a value is stored in a variable or written to the pipe (so later mutation does not alias); also notes types with instance members. | names | assignment to object-typed variables, pipe writes | 5 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSVariableAssignmentBinder@L1868-L2110` | +| `PSBinaryOperationBinder` | `BinaryOperationBinder` | Binary operators on arbitrary operands: dispatch on the operation, then rules per operand types (numeric promotion, string/array/hashtable arithmetic, comparison with enumerable filtering, user-defined op_* methods). See OPERATOR-TABLE.md. | read | every non-native binary operator | 25 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422` | +| `PSUnaryOperationBinder` | `UnaryOperationBinder` | Unary Not, OnesComplement (-bnot), plus, minus, increment, decrement (members Not, BNot, UnaryPlus, UnaryMinus, IncrDecr). | names | unary operators | 10 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSUnaryOperationBinder@L3427-L3738` | +| `PSConvertBinder` | `ConvertBinder` | Cast/conversion: AutomationNull converts to the default value of the target; otherwise FigureConversion(value, type) and invoke the converter with invariant culture; binding is restricted by a version number and the language mode; ThrowNoConversion raises the cast error. | read | `[type]value`, typed variables, parameter conversions | 16 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSConvertBinder@L3743-L3941` | +| `PSGetIndexBinder` | `GetIndexBinder` | Index read: dictionaries, arrays (negative indices count from the end, multi-dimensional arrays, slicing by index lists), strings, objects with an indexer, null-conditional; errors via CannotIndexTarget. | names | `$x[i]`, `?[` | 7 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSGetIndexBinder@L3946-L4533` | +| `PSSetIndexBinder` | `SetIndexBinder` | Index write with the same shapes as read, plus value conversion. | names | `$x[i] = v` | 4 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSSetIndexBinder@L4538-L4905` | +| `PSGetMemberBinder` | `GetMemberBinder` | Property/field read: instance (ETS) members, adapted .NET members, hashtable/dictionary keys, member enumeration over collections, reserved intrinsic members; strict-mode property-missing errors; language-mode checks. | names | `$x.Name`, `[T]::Name` | 25 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSGetMemberBinder@L4910-L5956` | +| `PSGetMemberBinder.ReservedMemberBinder` | `PSGetMemberBinder` | Handles intrinsic reserved members of PSObject-style access (nested binder). | names | reserved member names | 5 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::ReservedMemberBinder@L4936-L4974` | +| `PSSetMemberBinder` | `SetMemberBinder` | Property/field write with value conversion to the member type, instance members and adapted members. | names | `$x.Name = v` | 4 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSSetMemberBinder@L5961-L6594` | +| `PSInvokeBinder` | `InvokeBinder` | Invoke a callable value. | names | invocation of delegates/script blocks | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSInvokeBinder@L6596-L6606` | +| `PSInvokeMemberBinder` | `InvokeMemberBinder` | Method call: overload resolution (FindBestMethod with conversion ranks), .Where()/.ForEach() intrinsic methods on collections, method call on each element of a collection, parameterized properties, COM, member-invocation logging. | names | `$x.Method(args)`, `[T]::Method(args)` | 4 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSInvokeMemberBinder@L6608-L7737` | +| `PSCreateInstanceBinder` | `CreateInstanceBinder` | Constructor call for `[T]::new(args)` / `New-Object`-style creation; rejects by-ref-like and (optionally) non-public types. | names | `[T]::new(…)` | 3 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSCreateInstanceBinder@L7739-L7932` | +| `PSInvokeBaseCtorBinder` | `InvokeMemberBinder` | Call a base-class constructor from a class constructor. | names | `: base(args)` in class constructors | 2 | `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSInvokeBaseCtorBinder@L7934-L8012` | + +Rows: **30** (`GENERATED`). Binders without annotation: none. + +## 4. Common mechanics [IMPL] + +- **Defer when values are missing**: a binder first checks `target.HasValue`; if an operand is a `PSObject` wrapper with a different base object it defers to a PSObject-aware path (`DeferForPSObject`), except `Add` on enumerables. +- **Restrictions**: each produced rule is guarded by type restrictions (`PSGetTypeRestriction`) and, for conversions, by a binder *version* and the language mode, so a language-mode change or a type-table change invalidates cached rules (`InvalidateCache`, version counters; `ExecutionContext.LanguageMode` setter). +- **Per-site caches**: each emitted dynamic expression has its own `CallSite`; both execution tiers use them ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md) EQ-02). +- **Fallback with error suggestion**: binders pass an `errorSuggestion` so that "operation not supported" throws a PowerShell error instead of a DLR error. + +## 5. Guidance for MicroteamShell [LESSON] + +- **LSN-BIND-01** [ADAPT] (area: NLD-RUN) Replace DLR binders by explicit dispatch functions per operation (`Add(left, right)`, `Convert(value, type)`, `GetMember(target, name)`, `Index(target, args)`) with a small per-site cache only if profiling shows a need. Why: the DLR is a mechanism, the observable behavior is the rule set. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`. +- **LSN-BIND-02** [REPLICATE] (area: NLD-TYP) Keep the dedicated helper operations (enumerate, to-object-array, write-to-pipe, array-assignment shaping, pipeline-result-to-bool); they encode language rules. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder@L518-L780`, `PSPipeWriterBinder@L841-L926`, `PSArrayAssignmentRHSBinder@L935-L1042`, `PSPipelineResultToBoolBinder@L1119-L1178`. +- **LSN-BIND-03** [AVOID] (area: NLD-RUN) Do not port COM binders and `PSGetMemberBinder`'s COM/ETS-adapter paths in the first versions. Replace with a plain reflection member lookup. Evidence: `src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComBinder@L1-L378`. DEV: DEV-BIND-050 + +Verification: binder list reconciles with `data/binders.json` (counts above); `PSEnumerableBinder`, `PSToObjectArrayBinder`, `PSPipeWriterBinder`, `PSArrayAssignmentRHSBinder`, `PSPipelineResultToBoolBinder`, `PSConvertBinder.FallbackConvert`, `PSBinaryOperationBinder` (dispatch, add, multiply, divide/remainder, equality, numeric) bodies were read; the others by member list and compiler use. + diff --git a/docs-out/02-language/CONVERSION-RULES.md b/docs-out/02-language/CONVERSION-RULES.md new file mode 100644 index 0000000..fc4b2c6 --- /dev/null +++ b/docs-out/02-language/CONVERSION-RULES.md @@ -0,0 +1,202 @@ +--- +id: LANG-CONV-001 +title: Conversion rules +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-CNV-001] +sources: [src/System.Management.Automation/engine/LanguagePrimitives.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/enums.json, data/members-tierA.json] +--- + +# Conversion rules + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Where conversions are used: operators ([OPERATOR-TABLE.md](OPERATOR-TABLE.md)), casts `[type]value`, typed variables and parameters ([SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md), parameter binding in T33), and comparison. Overview: [02.6-types-and-operators.md](02.6-types-and-operators.md). + +## 1. Purpose and scope [SPEC] + +The conversion algorithm of `LanguagePrimitives.ConvertTo`: how the engine chooses a converter for (source type, target type), in which order the candidates are tried, which rank each gets, and the behavior of the main converters (string ↔ number, bool, arrays, enums, null, objects). Rank values drive overload resolution in method calls as well. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives@L298-L5885 | static class | the converter library (≈5,900 lines; conversion core read, see §19) | A | +| src/System.Management.Automation/engine/LanguagePrimitives.cs::ConversionRank@L251-L292 | enum | quality of a conversion | A | +| src/System.Management.Automation/engine/LanguagePrimitives.cs::IConversionData@L4371-L4383 | interface | a cached converter plus its rank | A | + +## 3. Responsibilities and non-responsibilities + +Chooses and runs converters; does not bind parameters (T33) or format output. Member/method binding rules that use these ranks live in the binders (T26). + +## 4. Key types and contracts [IMPL] + +- `ConvertTo(value, type)` (public) uses `CultureInfo.InvariantCulture`; internal overload `ConvertTo(value, type, recursion, formatProvider, backupTypeTable)` finds `FigureConversion(value, type, out debase)` and invokes it, passing the base object when the source is a `PSObject` that should be unwrapped (src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.ConvertTo@L1670-L1673). +- `FigureConversion(fromType, toType)` returns an `IConversionData` (converter + `ConversionRank`) and caches it per type pair in a process-wide cache (`CacheConversion`, `GetConversionData`, `RebuildConversionCache` pre-populates numeric ↔ string/char/bool conversions). +- `IsTrue(object)` is the truthiness function (§7.3). +- `Equals`/`Compare` implement the comparison operators' non-numeric fallbacks (§7.6). + +## 5. Data model — ranks + +Higher rank = better match. `…S2A` variants are the same conversion applied to a scalar that is wrapped into an array. `ValueDependent` marks conversions whose applicability depends on the value (e.g. scalar to array). + + +| Rank name | Value | Source line | +|---|---|---:| +| `None` | `0x0000` | L253 | +| `UnrelatedArraysS2A` | `0x0007` | L254 | +| `UnrelatedArrays` | `0x000F` | L255 | +| `ToStringS2A` | `0x0017` | L256 | +| `ToString` | `0x001F` | L257 | +| `CustomS2A` | `0x0027` | L258 | +| `Custom` | `0x002F` | L259 | +| `IConvertibleS2A` | `0x0037` | L260 | +| `IConvertible` | `0x003F` | L261 | +| `ImplicitCastS2A` | `0x0047` | L262 | +| `ImplicitCast` | `0x004F` | L263 | +| `ExplicitCastS2A` | `0x0057` | L264 | +| `ExplicitCast` | `0x005F` | L265 | +| `ConstructorS2A` | `0x0067` | L266 | +| `Constructor` | `0x006F` | L267 | +| `Create` | `0x0073` | L268 | +| `ParseS2A` | `0x0077` | L269 | +| `Parse` | `0x007F` | L270 | +| `PSObjectS2A` | `0x0087` | L271 | +| `PSObject` | `0x008F` | L272 | +| `LanguageS2A` | `0x0097` | L273 | +| `Language` | `0x009F` | L274 | +| `NullToValue` | `0x00AF` | L275 | +| `NullToRef` | `0x00BF` | L276 | +| `NumericExplicitS2A` | `0x00C7` | L277 | +| `NumericExplicit` | `0x00CF` | L278 | +| `NumericExplicit1S2A` | `0x00D7` | L279 | +| `NumericExplicit1` | `0x00DF` | L280 | +| `NumericStringS2A` | `0x00E7` | L281 | +| `NumericString` | `0x00EF` | L282 | +| `NumericImplicitS2A` | `0x00F7` | L283 | +| `NumericImplicit` | `0x00FF` | L284 | +| `AssignableS2A` | `0x0107` | L285 | +| `Assignable` | `0x010F` | L286 | +| `IdentityS2A` | `0x0117` | L287 | +| `StringToCharArray` | `0x011A` | L288 | +| `Identity` | `0x011F` | L289 | +| `ValueDependent` | `0xFFF7` | L291 | + + +Rows: 38 (`GENERATED` from `data/enums.json`). + +## 6. Control flow + +```mermaid +flowchart TD + A[ConvertTo value, type] --> B[FigureConversion fromType, toType] + B --> C{cached?} + C -->|yes| R[run converter] + C -->|no| D{source is null?} + D -->|yes| N[null rules] + D -->|no| E{assignable?} + E -->|yes| I[identity or assignable] + E -->|no| F[PSObject rules] + F --> G[constrained language gate] + G --> H[language conversions] + H --> P[Parse then Create then constructor then cast then IConvertible then IDictionary] + P --> Q[IEnumerable constructor then property conversion] + Q --> T[custom TypeConverter wrap] + T --> X[no conversion] +``` + +## 7. Algorithms + +### 7.1 Converter selection (CF-CNV-001) + +**Conversion algorithm and converter order.** For a source type S and target type T the engine returns the first applicable converter in this order (cached per type pair): (1) a cached entry; (2) if S is the null type: `Nullable<…>` → null, any reference type → null, a value type → no conversion (numeric, `char`, `string`, `bool`, `PSReference`, `switch` and `void` targets have pre-cached null conversions); (3) if T is assignable from S: identity (same type) or assignable; (4) by-ref-like types: none; (5) S is a `PSObject`: none (the caller unwraps first); (6) T is `PSObject`: wrap; (7) under Constrained Language Mode: only targets `object`, `object[]` or the core type list are converted (others are refused, or only audited in WDAC audit mode); (8) **language conversions**: `Nullable` through U, `void`, `bool` (from `IList`, enum, value type, class), `string`, arrays (related arrays element-wise, unrelated arrays per element, enumerable to array, scalar to one-element array), `Array`, `Hashtable` from `IDictionary`, `PSReference`, `XmlDocument`, `StringCollection`, `ScriptBlock` to delegate, `PSCustomObject`, integer to enum, `PSMethod` to delegate; (9) a static `Parse` method on T; (10) a static `Create` method; (11) a constructor of T taking S; (12) a cast operator (implicit, then explicit) on S or T; (13) `IConvertible` (numeric sources through the string-convertible path, other `IConvertible` sources via `Convert`); (14) for an `IDictionary` source: T's no-argument constructor then property assignment from the keys; (15) an `IEnumerable` constructor; (16) property-based conversion; if S or T may have a `TypeConverter`/`PSTypeConverter`, or a value-dependent converter was found, the result is wrapped to try the custom/value-dependent converter first and the found converter second; (17) otherwise no conversion (rank `None`, a failing converter that raises `PSInvalidCastException`). Numeric-to-numeric, number-to-string, number-to-bool and null-to-number conversions are pre-cached with fixed ranks. + +Evidence: `src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828`, `LanguagePrimitives.FigureLanguageConversion@L4966`, `LanguagePrimitives.FigureConversionFromNull@L5830-L5848`, `LanguagePrimitives.RebuildConversionCache@L4468`, src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.ConvertTo@L1670-L1673. + +### 7.2 String ↔ number + +- **String → number** (`ConvertStringToInteger/Decimal/Real`): empty string → 0; otherwise the text is scanned with the **language number parser** (`Parser.ScanNumber(text, type, shouldTryCoercion: false)`), so hexadecimal (`0x`), binary, type suffixes and multipliers (`kb`…) are accepted exactly as in the tokenizer (CF-LEX-002 in [02.1-tokenizer.md](02.1-tokenizer.md)); the result is converted to the target type with invariant culture; if that fails, a type-converter parse is tried, and on `FormatException` a conversion through `double`; any remaining failure raises `PSInvalidCastException` (`InvalidCastFromStringToInteger`/`…ToDecimal`). +- **Number → string** (`ConvertNumericToString`): if the value came from a numeric literal the original **token text** is used; otherwise `double` uses format `G15`, `float` uses `G7`, `BigInteger` and the others use invariant culture formatting. +- **Other → string** (`ConvertNonNumericToString`): `PSObject.ToStringParser(context, value, formatProvider)` (honors `ToString` overrides and `$OFS` for collections, per ETS). + +### 7.3 Truthiness (`IsTrue`) + +`$null` and `AutomationNull` → false; after unwrapping `PSObject`: `bool` as is; `string` → true iff non-empty; numeric types → true iff non-zero; `SwitchParameter` → its value; `IList`: 0 elements → false, 1 element → truthiness of that element (if that element is itself a list: true iff it is non-empty), more than 1 → true; every other object → true. Evidence: src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1017. + +### 7.4 Enums + +String → enum: case-sensitive parse, then case-insensitive, then **unique-prefix disambiguation** (`EnumMinimumDisambiguation`); an undefined numeric value is an error; integer → enum is a language conversion; an enumerable of strings is joined with commas then parsed (flags). + +### 7.5 Arrays and hashtables + +Related arrays (element type assignable): element-wise; unrelated arrays: each element converted; any enumerable → array of converted elements; a scalar convertible to the element type → one-element array (value-dependent, ranks with `S2A`); `Hashtable` from any `IDictionary` copies entries. + +### 7.6 Equality and comparison fallbacks + +`LanguagePrimitives.Equals(first, second, ignoreCase, culture)`: unwrap `PSObject`; null equals only null; if `first` is a string the second is converted to string and compared with the culture's `CompareInfo` (ignore-case option); `first.Equals(second)`; both numeric → numeric comparison; `char` with ignore-case compares upper-cased characters; otherwise convert `second` to `first`'s type and `Equals`, an `InvalidCastException` giving false. `Compare` follows the same shape (null sorts before numbers by sign rule, string by culture compare, numeric compare, else convert the second operand to the first's type and use `IComparable`; failures raise `ComparisonFailure`; non-comparable → `NotIcomparable`). Evidence: src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.Equals@L599-L602, src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.Compare@L742-L745. + +## 8. Observable behavior [SPEC] + +Read from source: `[int]"0x1F"` → 31; `[int]""` → 0; `[string]1.5` → "1.5"; `[bool]"false"` → true (non-empty string); `[bool]@()` → false; `[bool]@(0)` → false (one element: truthiness of `0`); `[bool]@(1,2)` → true; `[int[]]"1"` → one-element array. No golden sample: STATIC-ONLY. Tests: `test/powershell/Language/Operators/` and type-conversion tests (`data/tests.json`, mapping `UNVERIFIED`). + +## 9. Error handling and edge cases + +Failures raise `PSInvalidCastException` (message ids such as `InvalidCastFromStringToInteger`) or `PSArgumentException` for comparison failures. A converter exception unwraps its inner exception first for string → number. Conversion of `null` to a value type without a pre-cached null converter has rank `None`. `ConvertTo` returns `value` directly if it is already a `T`. + +## 10. Concurrency and threading + +The conversion cache is a process-wide dictionary guarded by a lock on rebuild (`lock (s_converterCache)`) and `ConcurrentDictionary`-style access for lookups (`INFERENCE` from `RebuildConversionCache`); a type-table-dependent cache is rebuilt when the type table changes. + +## 11. Extension points and customization hooks + +`PSTypeConverter`/`TypeConverter` attributes (checked in step 16), `ETS` type-table converters (`UpdateTypeConvertFromTypeTable`), static `Parse`/`Create`, constructors and cast operators on user types, `ArgumentTransformationAttribute` on variables and parameters (applied before this layer). + +## 12. Dependencies + +Inbound: binders (`PSConvertBinder`), cast expressions, typed variables, parameter binder, comparison operators. Outbound: `Parser.ScanNumber`, `PSObject`, `TypeTable`, reflection, `SystemPolicy`. + +## 13. Performance notes + +Converters are cached per type pair; dynamic call sites call the converter directly after a rank check; numeric conversions are pre-seeded to avoid reflection on first use. + +## 14. Security considerations (mandatory) + +Constrained Language Mode refuses conversions to arbitrary types (step 7) — the main gate against constructing disallowed .NET types through casts; WDAC audit mode logs instead. User-defined `Parse`/constructor/cast conversions run arbitrary code in Full Language mode. + +## 15. Enterprise considerations + +`typeConversion` trace source records each conversion step (`WriteLine`) for diagnostics (`Trace-Command -Name TypeConversion`, `INFERENCE`). Missing: no way to disable individual conversion steps. + +## 16. Tests + +Not read per assertion (VB-005). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| one ordered search over many conversion mechanisms | `FACT-SRC` | very permissive casts (`[type]` works for anything with a parse/ctor/cast) | +| ranks double as overload-resolution scores | `FACT-SRC` | single scoring scale for conversion and method binding | +| string → number reuses the language number parser | `FACT-SRC` | literal syntax and string conversion agree | +| invariant culture for ordinary conversions | `FACT-SRC` | scripts behave the same on every machine | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-CONV-01** [REPLICATE] (area: NLD-TYP) Use the search order of §7.1 with a per-(source,target) cache and an explicit rank for each outcome. Evidence: `src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828`. +- **LSN-CONV-02** [REPLICATE] (area: NLD-TYP) Convert strings to numbers with the same scanner as the tokenizer (hex, binary, suffixes, multipliers) and numbers to strings with invariant formatting (`G15`/`G7`, literal token text preserved). Evidence: src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.TryScanNumber@L2882-L2906, src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.ConvertNumericToString@L3322-L3363. +- **LSN-CONV-03** [REPLICATE] (area: NLD-TYP) Implement `IsTrue` exactly as in §7.3, including the one-element-list rule. Evidence: src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1017. +- **LSN-CONV-04** [ADAPT] (area: NLD-TYP) Support the language conversions (§7.1 step 8) and `Parse`/constructor/cast mechanisms for MicroteamShell's own types first; add reflection-based discovery of .NET `Parse`/`Create`/constructors only for the .NET types the shell exposes. Why: same observable casts for supported types without reflection over arbitrary assemblies. Evidence: src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureLanguageConversion@L4966-L5154. +- **LSN-CONV-05** [AVOID] (area: NLD-SEC) Do not copy the Constrained-Language gate or the WDAC audit hook in the conversion layer; use an allow-list of target types at the shell's type-system boundary if a restricted mode is wanted. Evidence: `src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828`. DEV: DEV-TYP-070 + +## 19. Open questions / UNVERIFIED + +- Not read: the value-based `FigureConversion` overload (debasing rules), `FigureStaticCreateMethodConversion`, `FigureParseConversion`, `FigureConstructorConversion`, `FigureCastConversion`, `FigurePropertyConversion`, and the null converters' bodies (names and order only). The statements in §7.1 steps 9-16 are therefore `FACT-SRC` for **order** and `UNVERIFIED` for **detail** (VB-018). +- Exact numeric widening/narrowing rules for numeric ↔ numeric casts (`ConvertNumeric`, `ConvertIConvertible`) and bool ↔ string-valued conversions at the edges. +- Culture handling when a non-invariant provider is passed. + +Verification: `LanguagePrimitives.cs` windows read: L449-L560 (enumerables), L599-L865 (Equals, Compare), L980-L1060 (IsTrue), L1670-L1770 and L4860-L4895 (ConvertTo), L2882-L3074 (string → number), L3096-L3135 and L3322-L3400 (void, bool, string conversions), L3581-L3660 (enum), L4468-L4520 (cache seeding), L4966-L5120 (language conversions), L5641-L5848 (FigureConversion, null); rank table generated from `data/enums.json`. diff --git a/docs-out/02-language/EQUIVALENCE-SPEC.md b/docs-out/02-language/EQUIVALENCE-SPEC.md index 1716f83..cfa52cf 100644 --- a/docs-out/02-language/EQUIVALENCE-SPEC.md +++ b/docs-out/02-language/EQUIVALENCE-SPEC.md @@ -123,7 +123,7 @@ None located for tier parity (VB-014). - **LSN-EQV-01** [REPLICATE] (area: NLD-RUN) Define behavior once, in the evaluator and the protocols, never per execution strategy; if two strategies are ever added, test them differentially. Evidence: src/System.Management.Automation/engine/interpreter/Interpreter.cs::Interpreter@L31-L109. - **LSN-EQV-02** [REPLICATE] (area: NLD-ERR) `finally` must run for every exit (return, break, continue, throw) and flow-control signals must never be swallowed by catch-all handlers. Evidence: src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterTryCatchFinallyInstruction@L305-L447. -- **LSN-EQV-03** [AVOID] (area: NLD-RUN) Do not port the typed arithmetic/comparison instruction families or the DLR-derived interpreter; they are unreachable or defective for PowerShell. Replace with the evaluator's own numeric semantics from CONVERSION-RULES (planned T28). Evidence: src/System.Management.Automation/engine/interpreter/Instruction.cs::Instruction@L27-L70. DEV: DEV-EQV-061 +- **LSN-EQV-03** [AVOID] (area: NLD-RUN) Do not port the typed arithmetic/comparison instruction families or the DLR-derived interpreter; they are unreachable or defective for PowerShell. Replace with the evaluator's own numeric semantics from [CONVERSION-RULES.md](CONVERSION-RULES.md). Evidence: src/System.Management.Automation/engine/interpreter/Instruction.cs::Instruction@L27-L70. DEV: DEV-EQV-061 ## 19. Open questions / UNVERIFIED diff --git a/docs-out/02-language/GRAMMAR.ebnf.md b/docs-out/02-language/GRAMMAR.ebnf.md index 304b30c..06f1a00 100644 --- a/docs-out/02-language/GRAMMAR.ebnf.md +++ b/docs-out/02-language/GRAMMAR.ebnf.md @@ -20,7 +20,7 @@ generated_from: [data/parser-methods.json] ## 1. Purpose and scope [SPEC] -An implementer-oriented grammar of the PowerShell language as accepted by `Parser.cs`. It describes **syntax**, not semantics; operator precedence levels are in [02.2-parser.md](02.2-parser.md) §7.2 (generated) and `OPERATOR-TABLE.md` (planned) (planned T28). Tokenizer-mode dependent lexical rules are in [02.1-tokenizer.md](02.1-tokenizer.md); this grammar uses the token names of [TOKEN-CATALOG.md](TOKEN-CATALOG.md). +An implementer-oriented grammar of the PowerShell language as accepted by `Parser.cs`. It describes **syntax**, not semantics; operator precedence levels are in [02.2-parser.md](02.2-parser.md) §7.2 (generated) and [OPERATOR-TABLE.md](OPERATOR-TABLE.md). Tokenizer-mode dependent lexical rules are in [02.1-tokenizer.md](02.1-tokenizer.md); this grammar uses the token names of [TOKEN-CATALOG.md](TOKEN-CATALOG.md). Notation: `::=` defines; `|` alternative; `[ x ]` optional; `{ x }` zero or more; `( … )` grouping; `'text'` a token by its text; `Name` a `TokenKind` (e.g. `Variable`, `Generic`); lower-case-hyphen names are non-terminals. `nl` = zero or more `NewLine` tokens (`SkipNewlines`); `nls` = zero or more `NewLine` or `Semi` tokens (`SkipNewlinesAndSemicolons`). "Mode" says which tokenizer mode is active while the production is parsed (C = Command, E = Expression, T = TypeName, S = Signature). Each production names its parser method; all methods are in `src/System.Management.Automation/engine/parser/Parser.cs` (line ranges in [02.2-parser.md](02.2-parser.md) §7.1). diff --git a/docs-out/02-language/HOWTO-extend-language.md b/docs-out/02-language/HOWTO-extend-language.md new file mode 100644 index 0000000..0588a00 --- /dev/null +++ b/docs-out/02-language/HOWTO-extend-language.md @@ -0,0 +1,152 @@ +--- +id: LANG-HOWTO-001 +title: How to extend the language +kind: guide +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/token.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/ast.cs, src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/LanguagePrimitives.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [] +--- + +# How to extend the language + +> STATIC-ONLY — behavior not run-verified. Each recipe lists, in order, the places in the PowerShell source that a change touches, derived from the documents of this study (each step links to its evidence), and traces one existing feature end to end. The lists are what PowerShell itself needs; MicroteamShell can drop the DLR, compiler and visitor-version steps. `INFERENCE` marks steps deduced from the code shape (nothing was built or run). + +## 1. Purpose and scope [SPEC] + +Practical guidance for adding syntax or semantics: (a) a new operator, (b) a new statement keyword, (c) a new AST node, (d) a new literal, (e) a new automatic variable, (f) a new type conversion. + +## 2. Source map + +The places named below, by stage: tokenizer (`token.cs`, `tokenizer.cs`, `CharTraits.cs`), parser (`Parser.cs`), AST (`ast.cs`, `AstVisitor.cs`, `PreOrderVisitor.cs`), analysis (`SemanticChecks.cs`, `VariableAnalysis.cs`, `SymbolResolver.cs`), constants (`ConstantValues.cs`, `SafeValues.cs`), compiler (`Compiler.cs`), runtime (`Binders.cs`, `MiscOps.cs`, `parserutils.cs`, `LanguagePrimitives.cs`), variables (`SpecialVariables.cs`). + +## 3. Responsibilities and non-responsibilities + +A guide, not a specification. It does not say whether a change is advisable. + +## 4. Key types and contracts [IMPL] + +Contracts that every change must keep: tables in `token.cs` stay the same length as `TokenKind` (a `DEBUG` static constructor asserts it); every AST node must be added to both visitor families and to the debug parent checker; `Copy`, `Accept` and `InternalVisit` must agree on children order ([02.3-ast.md](02.3-ast.md) §6). + +## 5. Data model + +Not applicable (procedural guide). + +## 6. Control flow — recipes + +### 6.1 (a) A new operator + +1. Add a `TokenKind` member with an explicit value (use a reserved slot in 105-118 or append) and rows in `s_staticTokenFlags` and `s_tokenText` (src/System.Management.Automation/engine/parser/token.cs::TokenKind@L18-L595, src/System.Management.Automation/engine/parser/token.cs::TokenTraits@L757-L1211); flags: `BinaryOperator` plus a `BinaryPrecedence*` level (and `CanConstantFold`/`DisallowedInRestrictedMode` as needed). For a dash operator also add the spelling to `_operatorText` and the kind to `s_operatorTokenKind` in the same position (src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L595-L5082); for a symbol operator add a case in `Tokenizer.NextToken`. +2. Parser: nothing for an existing precedence level (`BinaryExpressionRule` is table-driven); a new level needs a new `BinaryPrecedence*` value (src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950). +3. AST: nothing (`BinaryExpressionAst` validates the `BinaryOperator` trait); extend `StaticType` if the result type is fixed (src/System.Management.Automation/engine/parser/ast.cs::BinaryExpressionAst@L7502-L7617). +4. Compiler: a case in `VisitBinaryExpression` (dynamic site with an operation id, or a helper call) (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitBinaryExpression@L5687-L5977). +5. Runtime: a rule in `PSBinaryOperationBinder.FallbackBinaryOperation` and its operation methods, or a `ParserOps` helper (src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.FallbackBinaryOperation@L2179-L2238). +6. Type inference/completion (`TypeInferenceVisitor.VisitBinaryExpression`) and restricted-language flags (`INFERENCE`: completion is optional). +7. Tests and docs: operator table, token catalog. +*Trace:* `-shl`/`-shr` have kind values 97 and 98, appended after the earlier operators, with `Comparison` precedence and `CanConstantFold`; the compiler maps them to `LeftShift`/`RightShift` dynamic operations and the binder implements `Shift` ([TOKEN-CATALOG.md](TOKEN-CATALOG.md) §3, [COMPILATION-MAP.md](COMPILATION-MAP.md), src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.Shift@L2872-L2918). `??` (kind 102) followed the same path with its own precedence level 7. + +### 6.2 (b) A new statement keyword + +1. Tokenizer: `TokenKind` + `s_keywordText`/`s_keywordTokenKind` rows + flag rows (`Keyword`, `StatementDoesntSupportAttributes` if attributes are not allowed, `ScriptBlockBlockName` for block names) (src/System.Management.Automation/engine/parser/token.cs::TokenTraits@L757-L1211, src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L595-L5082). +2. Parser: a `case` in `StatementRule` and a `*Rule` method building the AST; set tokenizer modes if the body needs them (src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementRule@L1950-L2159). +3. AST class (recipe c). +4. Semantic: `SemanticChecks` visit (placement rules), `VariableAnalysis` flow-graph visit, `SymbolResolver` if types are involved (src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514, src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847). +5. Compiler: `Compiler.Visit…` ([COMPILATION-MAP.md](COMPILATION-MAP.md)). +6. Restricted language: `RestrictedLanguageChecker` visit; completion/type inference visits. +*Trace:* `clean` (kind 170, the last token kind) is a keyword with `ScriptBlockBlockName`; `NamedBlockListRule` parses it, `ScriptBlockAst` gained `CleanBlock`, the compiler emits a `` lambda and `InvokeWithPipeImpl` rejects it for plain invocation (src/System.Management.Automation/engine/parser/Parser.cs::Parser.NamedBlockListRule@L1727-L1857, src/System.Management.Automation/engine/parser/ast.cs::NamedBlockAst@L1864-L2026, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitScriptBlock@L2450-L2502). + +### 6.3 (c) A new AST node + +1. Class in `ast.cs` with constructor checks, parent links (`SetParent`), `Copy`, `Accept` and `InternalVisit` (pre/post action pattern) (src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420, src/System.Management.Automation/engine/parser/ast.cs::IfStatementAst@L4059-L4157 as template). +2. A `VisitX` in `ICustomAstVisitor` (or `ICustomAstVisitor2` for source compatibility), `DefaultCustomAstVisitor`, `AstVisitor` (or `AstVisitor2`), `AstSearcher`, the DEBUG `CheckAllParentsSet` (src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235, src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215). +3. Implementations in every visitor class that must handle it: `Compiler`, `VariableAnalysis`, `IsConstantValueVisitor`, `ConstantValueVisitor`, `IsSafeValueVisitor`, `GetSafeValueVisitor`, `RestrictedLanguageChecker`, `TypeInferenceVisitor`, `ScriptBlockToPowerShellChecker`, … (list: [AST-CATALOG.md](AST-CATALOG.md) §6). +4. Parser production that creates it; semantic checks. +*Trace:* `TernaryExpressionAst` (second-generation node): class with three required children; visited by `AstVisitor2`/`ICustomAstVisitor2` only; created by `ExpressionRule`; compiled to `Expression.Condition`; folded by `IsConstantValueVisitor` (src/System.Management.Automation/engine/parser/ast.cs::TernaryExpressionAst@L7404-L7497, src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpressionRule@L6696-L6790, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTernaryExpression@L5677-L5685). + +### 6.4 (d) A new literal + +1. Tokenizer: a scan routine reached from `NextToken` (or a new branch in `ScanNumberHelper` for numbers), producing a `Token` subclass or an existing kind (src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081). +2. Parser: `PrimaryExpressionRule` case creating a constant AST (src/System.Management.Automation/engine/parser/Parser.cs::Parser.PrimaryExpressionRule@L7228-L7299). +3. AST: `ConstantExpressionAst`/`StringConstantExpressionAst` (new `StringConstantType` if quoting differs). +4. Compiler: constants are handled generically (`VisitConstantExpression`). +5. Conversions: `Parser.ScanNumber` reuses the tokenizer, so string-to-number follows automatically ([CONVERSION-RULES.md](CONVERSION-RULES.md) §7.2). +*Trace:* the `n` BigInteger suffix: `NumberSuffixFlags.BigInteger`, `CharTraits.TypeSuffix` includes `n`, `ScanNumberHelper` accepts it, `TryGetNumberValue` produces `BigInteger` (src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882, [02.1-tokenizer.md](02.1-tokenizer.md) §7.1). + +### 6.5 (e) A new automatic variable + +1. `SpecialVariables`: name constant; for a **slot** variable also an `AutomaticVariable` enum member (order is fixed: slots 0-8 are asserted by `VariableAnalysis`), `AutomaticVariableTypes` and `AutomaticVariables` entries (src/System.Management.Automation/engine/SpecialVariables.cs::SpecialVariables@L22-L399, src/System.Management.Automation/engine/SpecialVariables.cs::AutomaticVariable@L401-L413). +2. Variable analysis seeds all automatic variables in its constructor, so slot variables are picked up from the arrays (src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847). +3. Where the value is set: `InvokeWithPipeImpl` (for `$_`, `$input`, `$this`, `$PSScriptRoot`, `$PSCommandPath`) or the command processor for others. +4. Non-slot engine variables are defined in the global scope at session start (not traced). +*Trace:* `$PSCommandPath` is automatic variable 8, typed `string`, set with `$PSScriptRoot` by `SetPSScriptRootAndPSCommandPath` before the clause runs (src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.SetPSScriptRootAndPSCommandPath@L1072-L1084). + +### 6.6 (f) A new type conversion + +1. Language-level: a branch in `FigureLanguageConversion` with a converter method and rank, or rely on a static `Parse`/constructor/cast on the type (src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureLanguageConversion@L4966-L5154). +2. Pre-seeded conversions go in `RebuildConversionCache`. +3. Restricted modes: add the target type to `CoreTypes` only if safe in Constrained Language Mode (src/System.Management.Automation/engine/parser/TypeResolver.cs::CoreTypes@L709-L838). +4. Type accelerator: `TypeAccelerators` table. +*Trace:* conversion of `ScriptBlock` to delegates is a `toType.IsSubclassOf(Delegate)` branch with `ConvertScriptBlockToDelegate` at rank `Language` ([CONVERSION-RULES.md](CONVERSION-RULES.md) §7.1). + +## 7. Algorithms + +Not applicable. + +## 8. Observable behavior [SPEC] + +Not applicable (no run). + +## 9. Error handling and edge cases + +Typical failure modes visible in the code: table length mismatches (`DEBUG` asserts), missing visitor overrides (default returns `null`/`Continue`, silently skipping the node), second-generation nodes invisible to old visitors ([02.3-ast.md](02.3-ast.md) §6.2). + +## 10. Concurrency and threading + +Not applicable. + +## 11. Extension points and customization hooks + +Beyond core changes: dynamic keywords (`DynamicKeyword` with `PreParse/PostParse/SemanticCheck`), type accelerators, `ETS` type data, argument transformation/validation attributes. + +## 12. Dependencies + +Documents: [02.1-tokenizer.md](02.1-tokenizer.md), [02.2-parser.md](02.2-parser.md), [02.3-ast.md](02.3-ast.md), [02.4-semantic-analysis.md](02.4-semantic-analysis.md), [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md). + +## 13. Performance notes + +Not applicable. + +## 14. Security considerations (mandatory) + +Every new operator, node or conversion must be classified for restricted modes: token flag `DisallowedInRestrictedMode`, `RestrictedLanguageChecker` visit, `IsSafeValueVisitor` allow-list decision, `CoreTypes` membership. Forgetting one silently allows the construct in data sections or Constrained Language. + +## 15. Enterprise considerations + +Add corpus snippets and tests for each change (operator tables and catalogs here are generated, so regenerate them). + +## 16. Tests + +Not applicable. + +## 17. Design decisions and trade-offs + +PowerShell's extension cost is high because a construct appears in ~10 tables and visitors; a new shell should centralize per-construct metadata. + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-HOWTO-01** [ADAPT] (area: NLD-TOOL) Keep one declaration per operator/keyword (spelling, kind, precedence, flags, evaluator) and generate the tables, instead of parallel arrays kept in sync by assertions. Why: same behavior, less drift. Evidence: src/System.Management.Automation/engine/parser/token.cs::TokenTraits@L757-L1211. +- **LSN-HOWTO-02** [ADAPT] (area: NLD-TOOL) Give every AST node one visitor interface with a default method; add a test that visits all nodes of a corpus with each visitor. Why: PowerShell's two-generation visitor scheme is only for compatibility. Evidence: src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235. +- **LSN-HOWTO-03** [REPLICATE] (area: NLD-SEC) Classify each new construct for restricted evaluation at the time it is added. Evidence: src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454. + +## 19. Open questions / UNVERIFIED + +- Steps marked `INFERENCE` (completion, type inference) were not traced through the completion code (Tier C). +- No build was run to check that the recipes compile. + +Verification: every step links to the source or document that supports it; the traces use `TokenKind` values from [TOKEN-CATALOG.md](TOKEN-CATALOG.md), the `Compiler.VisitScriptBlock` clause names, `SetPSScriptRootAndPSCommandPath`, and the `FigureLanguageConversion` delegate branch read in this study. diff --git a/docs-out/02-language/IR-SPEC.md b/docs-out/02-language/IR-SPEC.md index 20b43d9..7ab54e1 100644 --- a/docs-out/02-language/IR-SPEC.md +++ b/docs-out/02-language/IR-SPEC.md @@ -35,7 +35,7 @@ PowerShell has no bytecode of its own. Its IR is a `System.Linq.Expressions` tre ## 3. Responsibilities and non-responsibilities -Describes the tree shape, not the AST → tree rules (see [COMPILATION-MAP.md](COMPILATION-MAP.md)) and not binder behavior (T26, planned BINDER-CATALOG). +Describes the tree shape, not the AST → tree rules (see [COMPILATION-MAP.md](COMPILATION-MAP.md)) and not binder behavior ([BINDER-CATALOG.md](BINDER-CATALOG.md)). ## 4. Key types and contracts [IMPL] @@ -65,7 +65,7 @@ A `MutableTuple` whose type comes from variable analysis ([02.4-semantic-analysi - **Error-action protocol.** Every statement is wrapped so that a non-flow-control exception is passed to `ExceptionHandlingOps.CheckActionPreference(funcContext, e)`; if it returns, execution resumes at the next statement; flow-control exceptions (`FlowControlException`) are rethrown untouched. Detail: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.4. - **Position protocol.** Before each statement/condition the tree stores the sequence-point index into `funcContext._currentSequencePointIndex` and, if the debugger is active (`_debuggingMode > 0`), calls `OnSequencePointHit`. `UpdatePositionExpr` reduces to `DebugInfo` + that store for the JIT, and to `UpdatePositionInstruction` for the interpreter. - **Flow-control protocol.** `break`/`continue` to a loop in the same lambda are `Goto` labels; otherwise `BreakException(label)`/`ContinueException(label)` are thrown and caught by the enclosing loop's catch blocks; `return` is a `Return` to the lambda's label (or `ReturnException` inside trap lambdas); `exit` throws the exception from `PipelineOps.GetExitException`; `throw` converts via `ExceptionHandlingOps.ConvertToException`. -- **Dynamic operations.** Arithmetic other than double-double, comparisons, conversions, member/index/invoke operations, enumeration, and pipeline writes are `DynamicExpression` sites using the PowerShell binders (`PSBinaryOperationBinder`, `PSConvertBinder`, `PSGetMemberBinder`, …); each emitted site has its own `CallSite` (per-site caches). Binder semantics: BINDER-CATALOG (planned T26). +- **Dynamic operations.** Arithmetic other than double-double, comparisons, conversions, member/index/invoke operations, enumeration, and pipeline writes are `DynamicExpression` sites using the PowerShell binders (`PSBinaryOperationBinder`, `PSConvertBinder`, `PSGetMemberBinder`, …); each emitted site has its own `CallSite` (per-site caches). Binder semantics: [BINDER-CATALOG.md](BINDER-CATALOG.md). ## 6. Control flow diff --git a/docs-out/02-language/OPERATOR-TABLE.md b/docs-out/02-language/OPERATOR-TABLE.md new file mode 100644 index 0000000..94e7f74 --- /dev/null +++ b/docs-out/02-language/OPERATOR-TABLE.md @@ -0,0 +1,139 @@ +--- +id: LANG-OPTABLE-001 +title: Operator table +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-OP-001] +sources: [src/System.Management.Automation/engine/parser/token.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/runtime/Binding/Binders.cs, src/System.Management.Automation/engine/lang/parserutils.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/enums.json, data/tables.json] +--- + +# Operator table + +> STATIC-ONLY — behavior not run-verified. Rows are `GENERATED` by `docs-out/tools/gen_T28.py` from the token tables (`APPROX`) joined with hand annotations (`docs-out/tools/annotations/T28-operators.json`, `FACT-SRC`). Narrative: [02.6-types-and-operators.md](02.6-types-and-operators.md); conversions used by operators: [CONVERSION-RULES.md](CONVERSION-RULES.md); parsing: [02.2-parser.md](02.2-parser.md) §7.2. + +## 1. Purpose and scope [SPEC] + +One table for every operator token kind: spelling, role, precedence level, associativity, whether the compiler may fold it at compile time, and how it is evaluated. The precedence rule is canonical fact CF-OP-001 (stated in §3). + +## 2. Source map + +| path | role | +|---|---| +| `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752` | precedence and operator-class flags | +| `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950` | precedence climbing | +| `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422` | binary operation semantics | +| `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSUnaryOperationBinder@L3427-L3738` | unary operation semantics | +| `src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1760` | `-like`, `-match`, `-replace`, `-split`, `-join`, range helpers | + +## 3. Precedence and associativity (CF-OP-001) + +**Operator precedence and associativity.** Binary operators are parsed by precedence climbing and **all are left-associative**; from lowest to highest precedence the levels are: logical (`-and`, `-or`, `-xor`) = 1; bitwise (`-band`, `-bor`, `-bxor`) = 2; comparison group (all `-eq`/`-ne`/`-gt`/`-ge`/`-lt`/`-le`, `-like`, `-match`, `-replace`, `-contains`, `-in` families with their `i`/`c` variants, `-is`, `-isnot`, `-as`, `-split`, `-join`, `-shl`, `-shr`) = 5; null-coalescing `??` = 7; additive (`+`, `-`) = 9; multiplicative (`*`, `/`, `%`) = 10; format `-f` = 12; range `..` = 13. The comma (array-literal) operator binds tighter than every binary operator; the unary operators, casts and `++`/`--` bind tighter than the comma; the ternary `?:` (right-nested) and assignment (`=`, `+=`, `-=`, `*=`, `/=`, `%=`, `??=`, parsed at statement level with a statement on the right) bind looser than all binary operators; `&&` and `||` chain pipelines and are not expression operators. Operators spelled with a leading dash are recognized as operators only when the tokenizer is in expression mode. + +Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950` (reduce while new precedence is less than or equal), `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752` (level values), `Parser.ArrayLiteralRule@L6975-L7021`, `Parser.UnaryExpressionRule@L7029-L7226`, `Parser.ExpressionRule@L6696-L6790`, `Parser.PipelineChainRule@L5773-L5988`. + +## 4. Operator table + +Level = `BinaryPrecedence*` value (higher binds tighter); `fold` = the operator may be constant-folded; `restricted` = flagged `DisallowedInRestrictedMode`. Case: `i`-prefixed and plain forms ignore case, `c`-prefixed forms are case-sensitive (token texts of `i` forms are listed without the `i` for most operators, see [TOKEN-CATALOG.md](TOKEN-CATALOG.md) §3). + +| TokenKind | Text | Class | Level | Associativity | fold | restricted | Evaluation | Behavior (FACT-SRC) | +|---|---|---|---:|---|---|---|---|---| +| `And` | `-and` | Binary | 1 | left | yes | | AndAlso/OrElse over bool conversions (-and, -or); NotEqual over two bools (-xor) | operands converted with PowerShell truthiness (IsTrue); -and/-or short-circuit; -xor evaluates both | +| `Or` | `-or` | Binary | 1 | left | yes | | AndAlso/OrElse over bool conversions (-and, -or); NotEqual over two bools (-xor) | operands converted with PowerShell truthiness (IsTrue); -and/-or short-circuit; -xor evaluates both | +| `Xor` | `-xor` | Binary | 1 | left | yes | | AndAlso/OrElse over bool conversions (-and, -or); NotEqual over two bools (-xor) | operands converted with PowerShell truthiness (IsTrue); -and/-or short-circuit; -xor evaluates both | +| `Band` | `-band` | Binary | 2 | left | yes | | Dynamic site PSBinaryOperationBinder(And/Or/ExclusiveOr) | integer bitwise operation on int, uint, long or ulong chosen from the wider operand | +| `Bor` | `-bor` | Binary | 2 | left | yes | | Dynamic site PSBinaryOperationBinder(And/Or/ExclusiveOr) | integer bitwise operation on int, uint, long or ulong chosen from the wider operand | +| `Bxor` | `-bxor` | Binary | 2 | left | yes | | Dynamic site PSBinaryOperationBinder(And/Or/ExclusiveOr) | integer bitwise operation on int, uint, long or ulong chosen from the wider operand | +| `As` | `-as` | Binary | 5 | left | | yes | TypeOps.AsOperator | converts or returns $null | +| `Ccontains` | `-ccontains` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Ceq` | `-ceq` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(Equal/NotEqual) with ignore-case flag (i-variants and the plain operator ignore case, c-variants do not) | null handling; enumerable left operand filters (returns matching elements) unless scalar compare; string left operand: ordinal-culture string compare with right operand converted to string; char left operand ignoring case; numeric compare; otherwise Equals, then Equals after converting the right operand to the left operand's type (an InvalidCast gives false) | +| `Cge` | `-cge` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Cgt` | `-cgt` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Cin` | `-cin` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Cle` | `-cle` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Clike` | `-clike` | Binary | 5 | left | | | ParserOps.LikeOperator(context, position, left, right, operator) | wildcard match; array left operand filters; constant right operand pre-built to a WildcardPattern | +| `Clt` | `-clt` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Cmatch` | `-cmatch` | Binary | 5 | left | | yes | ParserOps.MatchOperator(context, position, left, right, notMatch, ignoreCase) | regex match; scalar left operand sets $Matches (hashtable, case-insensitive keys; numeric group names become int keys); array left operand filters elements | +| `Cne` | `-cne` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(Equal/NotEqual) with ignore-case flag (i-variants and the plain operator ignore case, c-variants do not) | null handling; enumerable left operand filters (returns matching elements) unless scalar compare; string left operand: ordinal-culture string compare with right operand converted to string; char left operand ignoring case; numeric compare; otherwise Equals, then Equals after converting the right operand to the left operand's type (an InvalidCast gives false) | +| `Cnotcontains` | `-cnotcontains` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Cnotin` | `-cnotin` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Cnotlike` | `-cnotlike` | Binary | 5 | left | | | ParserOps.LikeOperator(context, position, left, right, operator) | wildcard match; array left operand filters; constant right operand pre-built to a WildcardPattern | +| `Cnotmatch` | `-cnotmatch` | Binary | 5 | left | | yes | ParserOps.MatchOperator(context, position, left, right, notMatch, ignoreCase) | regex match; scalar left operand sets $Matches (hashtable, case-insensitive keys; numeric group names become int keys); array left operand filters elements | +| `Creplace` | `-creplace` | Binary | 5 | left | | yes | ParserOps.ReplaceOperator(context, position, left, right, ignoreCase) | right operand is an array of 1..2 elements (pattern, replacement); replacement string or script block (MatchEvaluator); more than 2 elements is an error | +| `Csplit` | `-csplit` | Binary/Unary | 5 | left | | yes | ParserOps.SplitOperator / UnarySplitOperator(context, position, left, right, ignoreCase) | regex split by default (SimpleMatch option for literal); optional limit and options; predicate script block form; unary form splits on whitespace | +| `Icontains` | `-icontains` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Ieq` | `-eq` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(Equal/NotEqual) with ignore-case flag (i-variants and the plain operator ignore case, c-variants do not) | null handling; enumerable left operand filters (returns matching elements) unless scalar compare; string left operand: ordinal-culture string compare with right operand converted to string; char left operand ignoring case; numeric compare; otherwise Equals, then Equals after converting the right operand to the left operand's type (an InvalidCast gives false) | +| `Ige` | `-ge` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Igt` | `-gt` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Iin` | `-iin` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Ile` | `-le` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Ilike` | `-ilike` | Binary | 5 | left | | | ParserOps.LikeOperator(context, position, left, right, operator) | wildcard match; array left operand filters; constant right operand pre-built to a WildcardPattern | +| `Ilt` | `-lt` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag | same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable) | +| `Imatch` | `-imatch` | Binary | 5 | left | | yes | ParserOps.MatchOperator(context, position, left, right, notMatch, ignoreCase) | regex match; scalar left operand sets $Matches (hashtable, case-insensitive keys; numeric group names become int keys); array left operand filters elements | +| `Ine` | `-ne` | Binary | 5 | left | | | Dynamic site PSBinaryOperationBinder(Equal/NotEqual) with ignore-case flag (i-variants and the plain operator ignore case, c-variants do not) | null handling; enumerable left operand filters (returns matching elements) unless scalar compare; string left operand: ordinal-culture string compare with right operand converted to string; char left operand ignoring case; numeric compare; otherwise Equals, then Equals after converting the right operand to the left operand's type (an InvalidCast gives false) | +| `Inotcontains` | `-inotcontains` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Inotin` | `-inotin` | Binary | 5 | left | | | GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare) | collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate | +| `Inotlike` | `-inotlike` | Binary | 5 | left | | | ParserOps.LikeOperator(context, position, left, right, operator) | wildcard match; array left operand filters; constant right operand pre-built to a WildcardPattern | +| `Inotmatch` | `-inotmatch` | Binary | 5 | left | | yes | ParserOps.MatchOperator(context, position, left, right, notMatch, ignoreCase) | regex match; scalar left operand sets $Matches (hashtable, case-insensitive keys; numeric group names become int keys); array left operand filters elements | +| `Ireplace` | `-ireplace` | Binary | 5 | left | | yes | ParserOps.ReplaceOperator(context, position, left, right, ignoreCase) | right operand is an array of 1..2 elements (pattern, replacement); replacement string or script block (MatchEvaluator); more than 2 elements is an error | +| `Is` | `-is` | Binary | 5 | left | | | TypeOps.IsInstance / TypeIs on the base object; -isnot negates | right operand is a type; PSObject/PSCustomObject special cases | +| `IsNot` | `-isnot` | Binary | 5 | left | | | TypeOps.IsInstance / TypeIs on the base object; -isnot negates | right operand is a type; PSObject/PSCustomObject special cases | +| `Isplit` | `-isplit` | Binary/Unary | 5 | left | | yes | ParserOps.SplitOperator / UnarySplitOperator(context, position, left, right, ignoreCase) | regex split by default (SimpleMatch option for literal); optional limit and options; predicate script block form; unary form splits on whitespace | +| `Join` | `-join` | Binary/Unary | 5 | left | | yes | ParserOps.JoinOperator / UnaryJoinOperator(context, position, left, right) | joins elements converted to strings with the separator; unary form joins with an empty separator | +| `Shl` | `-shl` | Binary | 5 | left | yes | | Dynamic site PSBinaryOperationBinder(LeftShift/RightShift) | integer shift; operand type int, uint, long or ulong chosen from the wider operand | +| `Shr` | `-shr` | Binary | 5 | left | yes | | Dynamic site PSBinaryOperationBinder(LeftShift/RightShift) | integer shift; operand type int, uint, long or ulong chosen from the wider operand | +| `QuestionQuestion` | `??` | Binary | 7 | left | | | Compiler.Coalesce helper | left operand unless it is $null | +| `Minus` | `-` | Binary/Unary | 9 | left | yes | | Dynamic site PSBinaryOperationBinder(Subtract) | numeric (wider type, overflow to double); string left operand converted to number; otherwise op_Subtraction | +| `Plus` | `+` | Binary/Unary | 9 | left | yes | | Dynamic site PSBinaryOperationBinder(Add) (double,double: native add) | null + x gives x; numeric + numeric uses the wider type with overflow to double; numeric + string converts the string to a number (Parser.ScanNumber); string + any (and char + any) concatenates using the right operand's string form; array/enumerable + x appends (array concatenation); hashtable + hashtable merges, hashtable + other is an error; otherwise a user-defined op_Addition | +| `Divide` | `/` | Binary | 10 | left | yes | | Dynamic site PSBinaryOperationBinder(Divide) | null / x treats null as 0; int/int gives an int when the division is exact, otherwise a double; division by zero raises a RuntimeException; string operands converted to numbers; otherwise op_Division | +| `Multiply` | `*` | Binary | 10 | left | yes | | Dynamic site PSBinaryOperationBinder(Multiply) | null * x is null; numeric; numeric * string converts the string; string * n repeats the string; array/enumerable * n repeats the elements; otherwise op_Multiply | +| `Rem` | `%` | Binary | 10 | left | yes | | Dynamic site PSBinaryOperationBinder(Modulo) | null % x treats null as 0; integer remainder; remainder by zero raises a RuntimeException; otherwise op_Modulus | +| `Format` | `-f` | Binary | 12 | left | | yes | StringOps.FormatOperator(format string, argument) | string.Format with the right operand as arguments (array elements spread) | +| `DotDot` | `..` | Binary | 13 | left | | yes | ParserOps.RangeOperator / range enumerator | integer ranges up and down; single-character operands give a character range | +| `Ampersand` | `&` | Special | — | — | | | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `Bnot` | `-bnot` | Unary | — | — | yes | | Dynamic site PSUnaryOperationBinder(OnesComplement) | integer bitwise complement | +| `Colon` | `:` | Special | — | — | | yes | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `ColonColon` | `::` | Special | — | — | | yes | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `Comma` | `,` | Unary | — | — | | | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `DivideEquals` | `/=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | +| `Dot` | `.` | Special | — | — | | yes | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `Equals` | `=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | +| `Exclaim` | `!` | Unary | — | — | yes | | Dynamic site PSUnaryOperationBinder(Not) | logical negation of truthiness | +| `MinusEquals` | `-=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | +| `MinusMinus` | `--` | Unary/PrefixOrPostfix | — | — | | yes | Compiler.CompileIncrementOrDecrement via IAssignableValue | prefix returns the new value; postfix returns the old value (null postfix result is 0); needs an assignable operand | +| `MultiplyEquals` | `*=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | +| `Not` | `-not` | Unary | — | — | yes | | Dynamic site PSUnaryOperationBinder(Not) | logical negation of truthiness | +| `Pipe` | `|` | Special | — | — | | | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `PlusEquals` | `+=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | +| `PlusPlus` | `++` | Unary/PrefixOrPostfix | — | — | | yes | Compiler.CompileIncrementOrDecrement via IAssignableValue | prefix returns the new value; postfix returns the old value (null postfix result is 0); needs an assignable operand | +| `PostfixMinusMinus` | `--` | Unary/PrefixOrPostfix | — | — | | yes | Compiler.CompileIncrementOrDecrement via IAssignableValue | prefix returns the new value; postfix returns the old value (null postfix result is 0); needs an assignable operand | +| `PostfixPlusPlus` | `++` | Unary/PrefixOrPostfix | — | — | | yes | Compiler.CompileIncrementOrDecrement via IAssignableValue | prefix returns the new value; postfix returns the old value (null postfix result is 0); needs an assignable operand | +| `QuestionDot` | `?.` | Special | — | — | | yes | Parser/tokenizer level | member access, static member access, invocation/call/dot-source; no expression-tree operator | +| `QuestionQuestionEquals` | `??=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | +| `RedirectInStd` | `<` | other | — | — | | yes | Redirection objects (FileRedirection, MergingRedirection) | stream redirection, not an expression operator | +| `Redirection` | `redirection` | other | — | — | | yes | Redirection objects (FileRedirection, MergingRedirection) | stream redirection, not an expression operator | +| `RemainderEquals` | `%=` | Assignment | — | — | | | CompileAssignment / ReduceAssignment | = stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null | + +Rows: **75** (`GENERATED`). Kinds without annotation: none. + +## 5. Notes + +- **Left-operand driven.** For binary operators the left operand's runtime type selects the rule; the right operand is converted to it where needed. `1 + "2"` is 3, `"1" + 2` is `"12"` (string concatenation). +- **`-eq`/`-ne` with arrays.** A collection on the left filters (returns matching elements); a `$null` on the left of `-eq` returns `$true` only for `$null`. +- **Constant folding** only for operators with the `CanConstantFold` flag; operators depending on execution context (`-split`, `-join`, `-like`) or culture (`-f`) are not folded (comment on `TokenFlags.CanConstantFold`). +- **`-and`/`-or`** short-circuit; **`-xor`** does not. **`-band`/`-bor`/`-bxor`/`-shl`/`-shr`** operate on integers of the wider operand type. + +## 6. Guidance for MicroteamShell [LESSON] + +- **LSN-OP-01** [REPLICATE] (area: NLD-OP) Use the precedence levels and left associativity above; keep the comma tighter than binary operators and put ternary and assignment below. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950`. +- **LSN-OP-02** [REPLICATE] (area: NLD-OP) Choose the operation by the left operand's type and convert the right operand to it (CONVERSION-RULES); keep the `i`/`c` case variants and the array-filtering behavior of comparison operators. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`. +- **LSN-OP-03** [ADAPT] (area: NLD-OP) Implement operators as ordinary functions with an explicit dispatch table instead of dynamic binding sites. Why: same observable results; no DLR. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.FallbackBinaryOperation@L2179-L2240`. + +Verification: operator rows generated from `s_staticTokenFlags`/`s_tokenText`; precedence levels cross-checked with the parser's reduce loop (`<=`) and with the generated table in [02.2-parser.md](02.2-parser.md) §7.2; evaluation text read from `PSBinaryOperationBinder` (`BinaryAdd`, `BinaryMultiply`, `BinarySubDivOrRem`, `CompareEQ`, `BinaryEqualityComparison`, `BinaryNumericOp`), `NumericOps.cs` (`IntOps`), `Compiler.VisitBinaryExpression` and `ParserOps` notes. `-shl`/`-shr`, bitwise and `-ge/-gt` internals and `PSUnaryOperationBinder` were read only at the dispatch level (`UNVERIFIED` below that). + diff --git a/docs-out/02-language/RUNTIME-OPERATIONS.md b/docs-out/02-language/RUNTIME-OPERATIONS.md new file mode 100644 index 0000000..8a10e6e --- /dev/null +++ b/docs-out/02-language/RUNTIME-OPERATIONS.md @@ -0,0 +1,193 @@ +--- +id: LANG-RTOPS-001 +title: Runtime operations +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs, src/System.Management.Automation/engine/runtime/Operations/ArrayOps.cs, src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs, src/System.Management.Automation/engine/runtime/Operations/StringOps.cs, src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs, src/System.Management.Automation/engine/runtime/Operations/ClassOps.cs, src/System.Management.Automation/engine/lang/parserutils.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/runtime/_folder.md] +generated_from: [data/types.json, data/members-tierA.json] +--- + +# Runtime operations + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. These are the helper classes that compiled code calls ([COMPILATION-MAP.md](COMPILATION-MAP.md), [IR-SPEC.md](IR-SPEC.md)); the binder rules that choose them are in [BINDER-CATALOG.md](BINDER-CATALOG.md). + +## 1. Purpose and scope [SPEC] + +Catalog and semantics of the static helper classes in `engine/runtime/Operations/*.cs` plus the operator helpers and flow-control exceptions in `engine/lang/parserutils.cs` (a file whose name does not say so, CON-010). Each helper implements one runtime rule of the language: error-action handling, pipeline invocation, redirections, enumeration, array/hashtable/string arithmetic, numeric overflow rules, type tests, switch matching, class method wrappers. + +## 2. Source map + +Generated list of every helper type (51 types, `GENERATED` from `data/types.json` and `data/members-tierA.json`): + + +| Type | File | Lines | Members (scan) | +|---|---|---|---:| +| `FlowControlException` | `parserutils.cs` | L28-L31 | 1 | +| `LoopFlowException` | `parserutils.cs` | L36-L67 | 5 | +| `BreakException` | `parserutils.cs` | L72-L88 | 3 | +| `ContinueException` | `parserutils.cs` | L93-L109 | 3 | +| `ReturnException` | `parserutils.cs` | L111-L119 | 2 | +| `ExitException` | `parserutils.cs` | L124-L139 | 3 | +| `ExitNestedPromptException` | `parserutils.cs` | L144-L146 | 0 | +| `TerminateException` | `parserutils.cs` | L151-L153 | 0 | +| `StopUpstreamCommandsException` | `parserutils.cs` | L160-L168 | 2 | +| `SplitOptions` | `parserutils.cs` | L176-L221 | 0 | +| `PowerShellBinaryOperator` | `parserutils.cs` | L225-L225 | 0 | +| `ParserOps` | `parserutils.cs` | L232-L1624 | 45 | +| `ParserOps.SplitImplOptions` | `parserutils.cs` | L398-L402 | 0 | +| `ParserOps.ReplaceOperatorImpl` | `parserutils.cs` | L963-L1035 | 7 | +| `CompareDelegate` | `parserutils.cs` | L1330-L1330 | 0 | +| `RangeEnumerator` | `parserutils.cs` | L1633-L1699 | 13 | +| `CharRangeEnumerator` | `parserutils.cs` | L1705-L1756 | 9 | +| `InterpreterError` | `parserutils.cs` | L1761-L1966 | 6 | +| `ScriptTrace` | `parserutils.cs` | L1970-L2008 | 2 | +| `ArrayOps` | `ArrayOps.cs` | L13-L301 | 9 | +| `Internal.SessionStateKeeper` | `ClassOps.cs` | L32-L95 | 4 | +| `Internal.ScriptBlockMemberMethodWrapper` | `ClassOps.cs` | L98-L269 | 13 | +| `Internal.ClassOps` | `ClassOps.cs` | L274-L374 | 7 | +| `PipelineOps` | `MiscOps.cs` | L29-L889 | 18 | +| `CommandRedirection` | `MiscOps.cs` | L893-L944 | 4 | +| `MergingRedirection` | `MiscOps.cs` | L946-L1061 | 4 | +| `FileRedirection` | `MiscOps.cs` | L1063-L1333 | 12 | +| `FunctionOps` | `MiscOps.cs` | L1337-L1366 | 1 | +| `ScriptBlockExpressionWrapper` | `MiscOps.cs` | L1368-L1389 | 4 | +| `ByRefOps` | `MiscOps.cs` | L1391-L1400 | 1 | +| `HashtableOps` | `MiscOps.cs` | L1402-L1458 | 2 | +| `ExceptionHandlingOps` | `MiscOps.cs` | L1460-L2197 | 20 | +| `ExceptionHandlingOps.CatchAll` | `MiscOps.cs` | L1462-L1462 | 0 | +| `ExceptionHandlingOps.HandlerSearchResult` | `MiscOps.cs` | L1467-L1481 | 5 | +| `TypeOps` | `MiscOps.cs` | L2199-L2434 | 8 | +| `SwitchOps` | `MiscOps.cs` | L2436-L2590 | 3 | +| `WhereOperatorSelectionMode` | `MiscOps.cs` | L2595-L2621 | 0 | +| `EnumerableOps` | `MiscOps.cs` | L2623-L3667 | 21 | +| `EnumerableOps.NonEnumerableObjectEnumerator` | `MiscOps.cs` | L3408-L3441 | 7 | +| `MemberInvocationLoggingOps` | `MiscOps.cs` | L3669-L3764 | 3 | +| `Boxed` | `NumericOps.cs` | L9-L13 | 2 | +| `IntOps` | `NumericOps.cs` | L15-L129 | 12 | +| `UIntOps` | `NumericOps.cs` | L131-L211 | 11 | +| `LongOps` | `NumericOps.cs` | L213-L310 | 11 | +| `ULongOps` | `NumericOps.cs` | L312-L395 | 11 | +| `DecimalOps` | `NumericOps.cs` | L397-L682 | 32 | +| `DoubleOps` | `NumericOps.cs` | L684-L883 | 18 | +| `CharOps` | `NumericOps.cs` | L885-L944 | 5 | +| `StringOps` | `StringOps.cs` | L15-L107 | 6 | +| `VariableOps` | `VariableOps.cs` | L15-L326 | 8 | +| `VariableOps.UsingResult` | `VariableOps.cs` | L287-L290 | 1 | + + +## 3. Responsibilities and non-responsibilities + +Implements run-time rules called from the IR. Does not decide which rule applies (binders) or how a command is found (command discovery, T32). + +## 4. Key types and contracts [IMPL] + +| Group | Types | Contract | +|---|---|---| +| Error handling | `ExceptionHandlingOps` | `CheckActionPreference(funcContext, exception)`, `FindMatchingHandler`, `ProcessTraps`, `ConvertToException`, `SuspendStoppingPipeline`/`RestoreStoppingPipeline` | +| Pipelines | `PipelineOps`, `CommandRedirection`, `MergingRedirection`, `FileRedirection` | `InvokePipeline(input, ignoreInput, commands, ast, redirections, funcContext)`, `InvokePipelineInBackground`, argument splatting, exit exceptions, interrupt checks | +| Enumeration | `EnumerableOps` | enumerators, output unrolling, array concatenation and repetition, `Compare` (array filter), `.Where()` / `.ForEach()` intrinsic methods, member access over collections | +| Arrays and strings | `ArrayOps`, `StringOps`, `CharOps` | repetition, slicing, multi-dimensional access, format operator, string comparison | +| Numbers | `IntOps`, `UIntOps`, `LongOps`, `ULongOps`, `DecimalOps`, `DoubleOps` | arithmetic and comparison with PowerShell overflow and zero-division rules | +| Variables | `VariableOps` | see [SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md) | +| Types and switch | `TypeOps`, `SwitchOps`, `HashtableOps`, `FunctionOps`, `ByRefOps` | `-is`/`-as`, switch clause matching, hashtable literal construction, function definition | +| Classes | `ClassOps`, `ScriptBlockMemberMethodWrapper`, `SessionStateKeeper` | class property validation, base calls, method bodies as script blocks | +| Operators | `ParserOps` | `-like`, `-match`, `-replace`, `-split`, `-join`, range, `-contains`, user-defined operator lookup | +| Flow control | `FlowControlException`, `LoopFlowException`, `BreakException`, `ContinueException`, `ReturnException`, `ExitException`, `ExitNestedPromptException`, `TerminateException`, `StopUpstreamCommandsException` | exceptions used for non-local control flow | + +## 5. Data model + +Flow-control exceptions: `LoopFlowException` carries a label; `MatchLabel` compares ordinal-ignoring-case and an **empty label matches any** loop; `BreakException`/`ContinueException` derive from it; `ReturnException` is internal; `ExitException` carries the exit argument. Boxed constants for `bool` and small `int`s avoid allocations (`Boxed`, caches -100..1000 in `ParserOps`). + +## 6. Control flow + +### 6.1 `CheckActionPreference` (statement-level error handling) + +Called from the per-statement catch block of compiled code ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.2). Steps (`FACT-SRC`, src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788): + +1. Unwrap `TargetInvocationException`; convert a non-`RuntimeException` to one (`ConvertToRuntimeException`), updating its error record with the current script position and history id. +2. Set `$Error`/error variables for the statement (`SetErrorVariables`) and set `$?` to `$false`. +3. Read `$ErrorActionPreference` (default `Continue`). If it is `Break` and this is a first-time exception, break into the debugger. +4. If the current block has trap handlers: run `ProcessTraps` (finds a handler by the inner exception type first, then by the outer; runs the handler lambda in a **new scope** with `$_` set to a new `ErrorRecord` and the other automatic variables copied; a `continue` in the trap means *SilentlyContinue*, a `break` means *Stop*; otherwise the action is queried from the exception). +5. Else, if the exception is of a kind that cannot be stopped, continued or ignored → rethrow. +6. Else if the preference is `Inquire` (and prompting is allowed) → ask the host. +7. `SilentlyContinue`/`Ignore` → return (execution resumes at the next statement). `Stop` → mark prompts suppressed and rethrow. Otherwise (`Continue`): if there is no trap and the exception came from a `throw` statement → rethrow; else report the error record to the error stream (`ReportErrorRecord`) and, if it cannot be reported, rethrow. + +This is the rule that makes ordinary errors in script statements non-terminating by default while `throw` and `-ErrorAction Stop` terminate. The terminating/non-terminating canonical fact CF-ERR-001 is owned by the errors document (planned T38). + +### 6.2 Collections and arrays + +`EnumerableOps.WriteEnumerableToPipe` unrolls one level and disposes the enumerator (CF-PIP-001, [02.6-types-and-operators.md](02.6-types-and-operators.md)). `ArrayOps.Multiply(array, times)`: `times == 1` returns the same array; 0 or empty gives an empty array; the result is built by repeated doubling copies; the element count must fit `int` (else an invalid-cast error); in RestrictedLanguage mode (data sections) the result may not exceed 1,024 elements. `ArrayOps.SlicingIndex` collects `indexer(target, i)` results for each index and drops `AutomationNull` results. `ArrayOps.AddObjectArray` appends by resizing (array `+` scalar returns a new array). + +### 6.3 Strings and hashtables + +`StringOps.Multiply(s, n)`: negative `n` throws `ArgumentOutOfRangeException` (a source TODO says it should be a runtime error); 0 or empty gives the empty string; in RestrictedLanguage mode the result may not exceed 1,024 characters. `StringOps.Add` is `string.Concat`. `HashtableOps.AddKeyValuePair` rejects a `$null` key (`InvalidNullKey`) and a duplicate key (`DuplicateKeyInHashLiteral`, key text truncated to 40 characters); `HashtableOps.Add(a, b)` builds a new `Hashtable` (or `OrderedDictionary` when the left side is ordered) with a culture-ignore-case comparer and `Add`s all keys, so a key present on both sides throws. **Quirk:** hashtable literals use an ordinal-ignore-case comparer (`Compiler.BuildHashtable`) while `+` uses the current-culture-ignore-case comparer. + +## 7. Algorithms + +Numeric operators: see [02.6-types-and-operators.md](02.6-types-and-operators.md) §7.2. Flow control: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.3-6.4. Operator helpers and their regex cache: [OPERATOR-TABLE.md](OPERATOR-TABLE.md). + +## 8. Observable behavior [SPEC] + +`$a = 1,2; $a * 2` gives a 4-element array; `"ab" * 2` gives `"abab"`; `@{a=1} + @{a=2}` throws; `@{a=1;A=2}` throws a duplicate-key error; a failing statement in a script continues unless the preference or a trap says otherwise. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`InterpreterError.NewInterpreterException` builds runtime errors whose ids and messages are `ParserStrings` keys ([ERROR-CATALOG.md](ERROR-CATALOG.md) §4); `NewBackupInterpreterException` falls back to a generic message when the resource lookup fails. `ScriptTrace.Trace` writes `Set-PSDebug -Trace` lines when the trace level is high enough. + +## 10. Concurrency and threading + +All helpers are static and stateless except caches: boxed-int cache, regex cache (≤1,000 per option set), `s_` caches in `EnumerableOps`; they rely on the execution context obtained from thread-local storage (`LocalPipeline.GetExecutionContextFromTLS`). + +## 11. Extension points and customization hooks + +None public; adding a runtime rule means adding a helper and calling it from the compiler or a binder ([HOWTO-extend-language.md](HOWTO-extend-language.md)). + +## 12. Dependencies + +Inbound: compiled code, binders. Outbound: execution context, command processors (pipelines), providers (redirection to files), debugger, `PSObject`. + +## 13. Performance notes + +Boxed constants and small caches; array repetition by doubling copies; `IntOps` returns boxed objects but avoids exceptions on overflow. + +## 14. Security considerations (mandatory) + +RestrictedLanguage mode caps `string * n` and array repetition at 1,024 to bound data-section evaluation; no such cap exists in Full Language mode. `MemberInvocationLoggingOps` logs member invocations when enabled. `ExceptionHandlingOps.CheckActionPreference` honors `Inquire` prompts through the host. `SwitchOps.ResolveFilePath` resolves the `-file` argument at run time. + +## 15. Enterprise considerations + +Error variables, `$?`, error records with script positions and history ids are produced here; `ScriptTrace` offers statement tracing. Missing: no cap on repeated sizes outside data sections. + +## 16. Tests + +Not read per assertion (VB-005). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| language rules as static helpers called from compiled code | `FACT-SRC` | rules testable in isolation; compiler stays small per node | +| non-local control flow by exceptions | `FACT-SRC` | simple across script-block boundaries, slow for deep unwinding | +| errors are reported and execution continues by default | `FACT-SRC` | shell-style resilience; `throw`/Stop needed to abort | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-RTOPS-01** [REPLICATE] (area: NLD-ERR) Implement the seven-step error-action algorithm of §6.1 as one function used by the statement runner. Evidence: src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788. +- **LSN-RTOPS-02** [REPLICATE] (area: NLD-PIP) Use label-carrying break/continue signals where an empty label matches any loop. Evidence: src/System.Management.Automation/engine/lang/parserutils.cs::LoopFlowException@L36-L67. +- **LSN-RTOPS-03** [REPLICATE] (area: NLD-OP) Hashtable literal construction rejects null and duplicate keys; `+` of hashtables creates a new table and throws on a shared key. Evidence: src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::HashtableOps@L1402-L1458. +- **LSN-RTOPS-04** [ADAPT] (area: NLD-OP) Cap string and array repetition in all modes (not only data sections). Why: PowerShell caps only in restricted mode; a personal shell should not be able to exhaust memory by accident. Evidence: src/System.Management.Automation/engine/runtime/Operations/StringOps.cs::StringOps@L15-L107, src/System.Management.Automation/engine/runtime/Operations/ArrayOps.cs::ArrayOps@L13-L301. +- **LSN-RTOPS-05** [AVOID] (area: NLD-RUN) Do not port the class-method wrapper machinery (`ScriptBlockMemberMethodWrapper`, `SessionStateKeeper`, Reflection.Emit class definition) in the first versions; classes are EXTENDED scope. Evidence: src/System.Management.Automation/engine/runtime/Operations/ClassOps.cs::ScriptBlockMemberMethodWrapper@L98-L269. DEV: DEV-RTOPS-051 + +## 19. Open questions / UNVERIFIED + +- Not read: `PipelineOps.InvokePipeline`, redirection classes, `EnumerableOps.Where/ForEach/MethodInvoker`, `TypeOps`, `SwitchOps`, `ClassOps` bodies (names and ranges only, VB-021). +- `ConvertToException` / `ReportErrorRecord` details; `QueryForAction` for traps. + +Verification: `ExceptionHandlingOps.CheckActionPreference` and `ProcessTraps` (L1699-L1908), `HashtableOps` (L1402-L1458), `ArrayOps` (L15-L112), `StringOps` (L15-L60), `EnumerableOps.WriteEnumerableToPipe` (L3623-L3640) were read; `parserutils.cs` from the earlier notes (W1-W7); the type table is generated. diff --git a/docs-out/02-language/SCOPE-AND-VARIABLES.md b/docs-out/02-language/SCOPE-AND-VARIABLES.md new file mode 100644 index 0000000..c91f00c --- /dev/null +++ b/docs-out/02-language/SCOPE-AND-VARIABLES.md @@ -0,0 +1,241 @@ +--- +id: LANG-SCOPE-001 +title: Scopes and variables +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-SCP-001] +sources: [src/System.Management.Automation/engine/VariablePath.cs, src/System.Management.Automation/engine/SessionStateScope.cs, src/System.Management.Automation/engine/SessionStateScopeAPIs.cs, src/System.Management.Automation/engine/SessionStateVariableAPIs.cs, src/System.Management.Automation/engine/ScopedItemSearcher.cs, src/System.Management.Automation/engine/ShellVariable.cs, src/System.Management.Automation/engine/SpecialVariables.cs, src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/auto-variables.json, data/enums.json] +--- + +# Scopes and variables + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Where the execution context and session state fit: [../03-runtime/execution-context-and-session-state.md](../03-runtime/execution-context-and-session-state.md). How the compiler avoids name lookups: [02.4-semantic-analysis.md](02.4-semantic-analysis.md) §7.2 and [TIERING-POLICY.md](TIERING-POLICY.md) §7.1. + +## 1. Purpose and scope [SPEC] + +Rules for variable names (`$x`, `$global:x`, `${env:PATH}`-style drive paths), the scope chain, reading and writing variables, variable options (`ReadOnly`, `Constant`, `Private`, `AllScope`), automatic and preference variables, dot-sourcing, and `$using:`. Scope handling for functions, aliases, drives and cmdlet tables follows the same chain and is only mentioned. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/VariablePath.cs::VariablePath@L29-L313 | class | parses a variable name into scope/drive qualifiers | A | +| src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope@L17-L2012 | class | one scope: variable table, tables for functions/aliases/cmdlets/drives, locals tuple, dotted scopes | A | +| src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal@L14-L323 | partial class | `NewScope`, `RemoveScope`, current scope | A | +| src/System.Management.Automation/engine/SessionStateVariableAPIs.cs::SessionStateInternal@L21-L1862 | partial class | get/set/new/remove variable entry points | A | +| src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297 | class | scope walk for a variable lookup | A | +| src/System.Management.Automation/engine/ShellVariable.cs::PSVariable@L15-L754 | class | a variable object (name, value, options, attributes) | A | +| src/System.Management.Automation/engine/ShellVariable.cs::ScopedItemOptions@L882-L915 | enum | `None`, `ReadOnly`, `Constant`, `Private`, `AllScope`, `Unspecified` | A | +| src/System.Management.Automation/engine/SpecialVariables.cs::SpecialVariables@L22-L399 | static class | names, types and order of automatic/preference variables | A | +| src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps@L15-L326 | static class | variable access called from compiled code | A | + +## 3. Responsibilities and non-responsibilities + +Responsible for variable naming, visibility, creation and assignment rules. Not responsible for fast-slot allocation (compiler), for function/alias lookup order (command discovery, T32), or for provider drive operations beyond the `drive:name` read/write redirect. + +## 4. Key types and contracts [IMPL] + +- `VariablePath(path)` classifies a name. Flags: `Local`, `Script`, `Global`, `Private`, `Variable`, `Function`, `DriveQualified`, `Unqualified` (src/System.Management.Automation/engine/VariablePath.cs::VariablePathFlags@L10-L24). Prefix matching is case-insensitive on `global:`, `local:`, `private:`, `script:` and `variable:` (the latter may be followed by another scope prefix, e.g. `variable:global:x`). If no scope word matches and the text contains a `:` after position 0, the path is **drive-qualified** (a provider path such as `env:PATH`); otherwise it is an unqualified variable. `UnqualifiedPath` is the text after the last scanned colon. `IsUnscopedVariable` is true when none of Local, Script, Global, Private, Function, DriveQualified is set. +- `SessionStateScope` has `Parent`, `ScopeOrigin`, `ScriptScope` (set to itself for a script scope, else inherited from the parent), `StrictModeVersion`, `LocalsTuple` (slot storage from the compiler) and a stack of `DottedScopes` (src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope@L17-L2012). +- `PSVariable`: `Name`, `Value`, `Options`, `Attributes` (validation/transformation), `Description`, `IsPrivate`, `IsAllScope`, `IsReadOnly`, `IsConstant`. + +## 5. Data model + +- Each scope holds a lazily created, case-insensitive dictionary of `PSVariable` plus the compiled **locals tuple** (typed slots, see the compiler docs). A variable can therefore live in a tuple slot (`LocalVariable`) or in the dictionary; lookups consult both (§7.1). +- The **global scope** is the root; its default variables are `$null`, `$false`, `$true` (marked `AllScope`). Every new scope copies, **by reference**, all parent variables marked `AllScope` into its own table (src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.AddSessionStateScopeDefaultVariables@L1756-L1777). +- Automatic variables with fixed tuple slots 0..8 (`AutomaticVariable` enum order): `$_`, `$args`, `$this`, `$input`, `$PSCmdlet`, `$PSBoundParameters`, `$MyInvocation`, `$PSScriptRoot`, `$PSCommandPath`; preference variables 9..16: `Debug`, `Verbose`, `Error`(Action), `WhatIf`, `Warning`, `Information`, `Confirm`, `Progress` (`PreferenceVariable` enum). Types of the nine automatic slots: object, object[], object, object, `PSScriptCmdlet`, `PSBoundParametersDictionary`, `InvocationInfo`, string, string (src/System.Management.Automation/engine/SpecialVariables.cs::SpecialVariables@L22-L399). The generated table below lists every name constant in `SpecialVariables.cs`. + + +| Field | Variable name | Line | References (APPROX) | +|---|---|---:|---:| +| `HistorySize` | `$MaximumHistoryCount` | L24 | 2 | +| `MyInvocation` | `$MyInvocation` | L28 | 3 | +| `OFS` | `$OFS` | L32 | 1 | +| `PSStyle` | `$PSStyle` | L36 | 1 | +| `OutputEncoding` | `$OutputEncoding` | L40 | 3 | +| `VerboseHelpErrors` | `$VerboseHelpErrors` | L44 | 1 | +| `LogEngineHealthEvent` | `$LogEngineHealthEvent` | L50 | 1 | +| `LogEngineLifecycleEvent` | `$LogEngineLifecycleEvent` | L54 | 1 | +| `LogCommandHealthEvent` | `$LogCommandHealthEvent` | L58 | 1 | +| `LogCommandLifecycleEvent` | `$LogCommandLifecycleEvent` | L62 | 1 | +| `LogProviderHealthEvent` | `$LogProviderHealthEvent` | L66 | 1 | +| `LogProviderLifecycleEvent` | `$LogProviderLifecycleEvent` | L70 | 1 | +| `LogSettingsEvent` | `$LogSettingsEvent` | L74 | 1 | +| `PSLogUserData` | `$PSLogUserData` | L78 | 0 | +| `NestedPromptLevel` | `$NestedPromptLevel` | L84 | 2 | +| `CurrentlyExecutingCommand` | `$CurrentlyExecutingCommand` | L88 | 3 | +| `PSBoundParameters` | `$PSBoundParameters` | L92 | 5 | +| `Matches` | `$Matches` | L96 | 3 | +| `LastExitCode` | `$LASTEXITCODE` | L100 | 7 | +| `PSDebugContext` | `$PSDebugContext` | L104 | 2 | +| `StackTrace` | `$StackTrace` | L108 | 3 | +| `FirstToken` | `$^` | L112 | 2 | +| `LastToken` | `$$` | L116 | 2 | +| `PSItem` | `$PSItem` | L122 | 5 | +| `Underbar` | `$_` | L123 | 6 | +| `Question` | `$?` | L127 | 4 | +| `Args` | `$args` | L131 | 4 | +| `This` | `$this` | L135 | 5 | +| `Input` | `$input` | L139 | 3 | +| `PSCmdlet` | `$PSCmdlet` | L143 | 2 | +| `Error` | `$error` | L147 | 4 | +| `EventError` | `$error` | L151 | 1 | +| `PathExt` | `$env:PATHEXT` | L155 | 0 | +| `PSEmailServer` | `$PSEmailServer` | L159 | 1 | +| `PSDefaultParameterValues` | `$PSDefaultParameterValues` | L163 | 2 | +| `PSScriptRoot` | `$PSScriptRoot` | L167 | 18 | +| `PSCommandPath` | `$PSCommandPath` | L171 | 10 | +| `PSSenderInfo` | `$PSSenderInfo` | L175 | 1 | +| `pwd` | `$PWD` | L187 | 1 | +| `Null` | `$null` | L191 | 7 | +| `True` | `$true` | L195 | 4 | +| `False` | `$false` | L199 | 5 | +| `PSModuleAutoLoading` | `$PSModuleAutoLoadingPreference` | L203 | 1 | +| `IsLinux` | `$IsLinux` | L209 | 1 | +| `IsMacOS` | `$IsMacOS` | L213 | 1 | +| `IsWindows` | `$IsWindows` | L217 | 1 | +| `IsCoreCLR` | `$IsCoreCLR` | L221 | 1 | +| `DebugPreference` | `$DebugPreference` | L229 | 5 | +| `ErrorActionPreference` | `$ErrorActionPreference` | L233 | 7 | +| `ProgressPreference` | `$ProgressPreference` | L237 | 5 | +| `VerbosePreference` | `$VerbosePreference` | L241 | 5 | +| `WarningPreference` | `$WarningPreference` | L245 | 5 | +| `WhatIfPreference` | `$WhatIfPreference` | L249 | 5 | +| `ConfirmPreference` | `$ConfirmPreference` | L253 | 5 | +| `InformationPreference` | `$InformationPreference` | L257 | 5 | +| `NativeArgumentPassing` | `$PSNativeCommandArgumentPassing` | L269 | 2 | +| `ErrorView` | `$ErrorView` | L273 | 1 | +| `PSSessionConfigurationName` | `$PSSessionConfigurationName` | L280 | 2 | +| `PSSessionApplicationName` | `$PSSessionApplicationName` | L288 | 2 | +| `ExecutionContext` | `$ExecutionContext` | L294 | 1 | +| `Home` | `$HOME` | L295 | 1 | +| `Host` | `$Host` | L296 | 1 | +| `PID` | `$PID` | L297 | 1 | +| `PSCulture` | `$PSCulture` | L298 | 1 | +| `PSHome` | `$PSHOME` | L299 | 1 | +| `PSUICulture` | `$PSUICulture` | L300 | 1 | +| `PSVersionTable` | `$PSVersionTable` | L301 | 1 | +| `PSEdition` | `$PSEdition` | L302 | 2 | +| `ShellId` | `$ShellId` | L303 | 1 | +| `EnabledExperimentalFeatures` | `$EnabledExperimentalFeatures` | L304 | 2 | + +`AutomaticVariable` (`src/System.Management.Automation/engine/SpecialVariables.cs::AutomaticVariable@L401-L413`): `Underbar`=0, `Args`=1, `This`=2, `Input`=3, `PSCmdlet`=4, `PSBoundParameters`=5, `MyInvocation`=6, `PSScriptRoot`=7, `PSCommandPath`=8, `NumberOfAutomaticVariables`=9 + +`PreferenceVariable` (`src/System.Management.Automation/engine/SpecialVariables.cs::PreferenceVariable@L415-L425`): `Debug`=9, `Verbose`=10, `Error`=11, `WhatIf`=12, `Warning`=13, `Information`=14, `Confirm`=15, `Progress`=16 + + + +## 6. Control flow + +```mermaid +flowchart TD + A[read variable name] --> B{unqualified or local or script or global or private?} + B -->|drive qualified| P[provider read: env: etc.] + B -->|global| G[search global scope only] + B -->|local or private| L[search current scope only] + B -->|script| S[search script scope only] + B -->|unqualified| W[walk current scope then parents] + W --> V{found?} + V -->|private in ancestor scope| W + V -->|found| R[return value] + V -->|no more scopes| U[null, or error in strict mode] +``` + +## 7. Algorithms + +### 7.1 Scope and lookup rules (CF-SCP-001) + +**Scope, lookup and assignment rules.** A variable name is parsed into an optional scope qualifier (`global:`, `local:`, `script:`, `private:`) or a provider drive (`name:`), and an unqualified name. **Reading:** an unqualified name is searched in the current scope and then in each parent scope up to the global scope, and the first match wins; a variable marked `Private` is skipped when it is found in any scope other than the current one; `global:`, `local:`/`private:` and `script:` search exactly one scope (the global scope, the current scope, the script scope respectively); inside one scope the compiled locals tuple (and any dot-sourced tuples) is checked before the variable table; a drive-qualified name is read through the provider. A name that is not found yields `$null`, or a `VariableIsUndefined` error under strict mode (version 2: always; version 1: except inside expandable strings). **Writing:** an unqualified or `local:`/`private:` assignment always targets the current scope (a parent's variable of the same name is never modified; a new local variable is created if the current scope has none), `script:` and `global:` target the script and global scope; assigning to an existing variable in the target scope keeps its object and replaces its value after running its attributes (transformation, then validation), and a `ReadOnly` or `Constant` variable cannot be written (`VariableNotWritable`; `-Force` can overwrite `ReadOnly` but never `Constant`). **New scopes:** every function or script block call (not dot-sourcing) creates a child scope whose parent is the caller's scope; the script scope is the nearest enclosing scope created for a script file, or the global scope; variables marked `AllScope` are shared by reference with every new scope. **Dot-sourcing** creates no scope: the callee's locals tuple is pushed onto the current scope's `DottedScopes` for the duration of the call. + +Evidence: src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297 (private skip), src/System.Management.Automation/engine/ScopedItemSearcher.cs::ScopedItemSearcher@L15-L239 (single-scope vs chain), src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.TryGetVariable@L324-L341, src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.TryGetLocalVariableFromTuple@L680-L692, src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.SetVariable@L401-L510, src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121 (unqualified → `CloneAndSetLocal`), src/System.Management.Automation/engine/SessionStateVariableAPIs.cs::SessionStateInternal.GetVariableItem@L532-L558, src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal.NewScope@L227-L243, src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L105-L110, src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.GetVariableValue@L181-L214. + +### 7.2 Assignment details (`VariableOps.SetVariableValue`) + +1. Non-variable path (drive) → provider write. +2. Unscoped path → `CloneAndSetLocal` (forced local). +3. Look the variable up (`GetVariableItem`, single scope for a local path). If missing: create a `PSVariable` with the attributes from a type-constraint (`[int]$x = …`), store it in the target scope, notify the debugger. +4. If present and attributes were supplied: reject `ReadOnly`/`Constant`; run `TransformValue`, then `IsValidValue` (a failure throws `ValidationMetadataException`); store the raw value; **replace** the variable's attributes with the new ones. +5. If present and no attributes: plain `Value` set (the property setter runs the variable's existing attributes and debugger write check). +6. In Constrained Language Mode, values assigned to `global:` variables (and `script:` variables in a module scope) are marked untrusted (`MarkObjectAsUntrustedForVariableAssignment`). + +### 7.3 Reading details (`VariableOps.GetVariableValue`) + +Found → `var.Value` (getter may run debugger read check). Not found → debugger read check, then strict-mode rule above, else `$null`. `GetAutomaticVariableValue(tupleIndex)` reads the automatic slot and maps `AutomationNull` to `$null`/strict error. + +### 7.4 `$using:` + +`$using:name` is rewritten before execution: the compiler asks `VariableOps.GetUsingValue(locals, key, index, context)`, which looks for the key (or, for old callers, the index) in `$PSBoundParameters.ImplicitUsingParameters` of the current locals tuple, then of every enclosing scope's `LocalsTuple` and dotted scopes. The values are supplied by whoever starts the remote or job script block (`ScriptBlockToPowerShell`, see [SCRIPT-BLOCKS.md](SCRIPT-BLOCKS.md)). + +### 7.5 Options + +`ReadOnly` — write blocked unless forced; `Constant` — never writable or removable (a constant cannot be created with `AllScope` removed, etc.); `Private` — visible only in the scope that owns it; `AllScope` — copied into every new scope; `Unspecified` — "leave unchanged". An `AllScope` variable cannot have the option removed later for aliases and functions (`...AllScopeOptionCannotBeRemoved` errors); for variables the code path for locals with different options raises `VariableNotWritableRare`. + +## 8. Observable behavior [SPEC] + +Read from source: `function f { $x = 2 }` leaves the caller's `$x` unchanged; `$global:x = 2` and `$script:x = 2` write the named scope; `$private:x` is invisible to child scopes; `. f` runs `f` in the caller's scope; `[int]$n = "5"` stores an int and later `$n = "abc"` fails validation. No golden sample: STATIC-ONLY. Tests: `test/powershell/Language/Scripting/` and `Operators/` (`data/tests.json`; mapping `UNVERIFIED`). + +## 9. Error handling and edge cases + +`VariableNotWritable`, `VariableNotWritableRare`, `ValidationMetadataException`, `VariableIsUndefined` (strict), `NonExistingVariableReference` (`[ref]$undefined`), `SessionStateUnauthorizedAccessException` for removals of constants. `fastPath` assignment into the global table is only allowed for the root scope. `VariablePath("")` throws `ArgumentException`. Unusual: `variable:` prefix followed by a scope prefix is accepted; the drive check requires a `:` at index > 0. + +## 10. Concurrency and threading + +Scope tables are plain dictionaries, accessed on the runspace's thread; compiled code is not re-entrant per scope. `AllScope` variable names known to the compiler are kept in a static concurrent dictionary ([02.4-semantic-analysis.md](02.4-semantic-analysis.md) §10). + +## 11. Extension points and customization hooks + +`PSVariable` subclasses (`LocalVariable`, `NullVariable`, custom variable types with getter/setter) and `ValidateArgumentsAttribute`/`ArgumentTransformationAttribute` on variables. New automatic variable: see [HOWTO-extend-language.md](HOWTO-extend-language.md). + +## 12. Dependencies + +Inbound: compiled code (`VariableOps`), cmdlets (`Get/Set/New-Variable`), providers (`Variable:` drive), `InvokeWithPipeImpl`. Outbound: `ExecutionContext`, `SessionStateInternal`, debugger, language mode, `MutableTuple`. + +## 13. Performance notes + +Slots avoid dictionary lookups; the dictionary is created lazily; `AllScope` copying is done at scope creation (a loop over parent variables), so many `AllScope` variables slow every call. + +## 14. Security considerations (mandatory) + +Constrained Language Mode marks assigned objects untrusted for `global:` and module `script:` variables; `CheckVariableChangeInConstrainedLanguage` runs when a variable is stored and Constrained Language has ever been used in the process; `Private` and `ReadOnly`/`Constant` options are the only protection of variables from callee code (a callee can still use `global:`). Evidence: src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.CheckVariableChangeInConstrainedLanguage@L1983-L2009. + +## 15. Enterprise considerations + +Debugger hooks on variable read/write; trace level 2 logs assignments (`Set-PSDebug -Trace 2`). Missing: no per-variable access log other than debugging. + +## 16. Tests + +Not read per assertion (VB-005). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| assignment is always local to the current scope | `FACT-SRC` | callee cannot modify caller variables unintentionally; explicit `script:`/`global:` needed | +| reading falls through to parents | `FACT-SRC` | dynamic scoping: callees see caller variables | +| `AllScope` copies the same object into every scope | `FACT-SRC` | shared mutable state for `$null/$true/$false` and some engine variables | +| slots + dictionary dual storage | `FACT-SRC` | speed at the price of two lookup paths | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-SCOPE-01** [REPLICATE] (area: NLD-SCP) Dynamic scoping for reads (walk to parents, skip `private`), local-only writes, explicit `script:`/`global:`/`local:`/`private:` qualifiers. Evidence: src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297, src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121. +- **LSN-SCOPE-02** [REPLICATE] (area: NLD-SCP) Each function/script-block call creates a child scope; dot-sourcing runs in the caller's scope; a script file is a script scope. Evidence: src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal.NewScope@L227-L243. +- **LSN-SCOPE-03** [REPLICATE] (area: NLD-SCP) Type constraints on assignment (`[int]$x = …`) attach transformation/validation attributes to the variable and replace the previous ones on re-declaration. Evidence: src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121. +- **LSN-SCOPE-04** [REPLICATE] (area: NLD-SCP) Support `ReadOnly`, `Constant`, `Private`, `AllScope`, with the write rules of §7.1. Evidence: src/System.Management.Automation/engine/ShellVariable.cs::ScopedItemOptions@L882-L915. +- **LSN-SCOPE-05** [ADAPT] (area: NLD-SCP) Keep one variable store per scope (no tuple slots) in the first implementation, but keep the lookup API narrow so slots can be added. Why: same observable behavior; slots are an optimization. Evidence: src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.TryGetLocalVariableFromTuple@L680-L692. +- **LSN-SCOPE-06** [REPLICATE] (area: NLD-SCP) Strict-mode version is found by walking scopes, stopping at the module scope; version 0 means off. Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L105-L110. +- **LSN-SCOPE-07** [AVOID] (area: NLD-SEC) Do not implement the untrusted-value marking tied to Constrained Language Mode unless a language-mode feature is adopted. Evidence: src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121. DEV: DEV-SCP-080 + +## 19. Open questions / UNVERIFIED + +- Exact behavior of `PSVariable.Value` setter and getter (attribute transformation order with `ValidateSet`) was not read line by line (VB-017). +- `Set-Variable -Scope N` numeric scope indexing and `Get-Variable` listing rules were not analyzed. +- Behavior of dotted scopes with `private:` variables. + +Verification: `VariablePath.cs` (322 lines) and `VariableOps.cs` (327) read in full; `SessionStateScope` (`TryGetVariable`, `SetVariable`, `TryGetLocalVariableFromTuple`, default variables), `ScopedItemSearcher` (variable searcher), `SessionStateVariableAPIs` (`GetVariableItem`, `SetVariable`), `SessionStateScopeAPIs.NewScope`, `ExecutionContext.IsStrictVersion` read in targeted windows (notes `.work/notes/T29-scope.md`); variable name table generated from `data/auto-variables.json` (`SPECIAL_VARIABLE_CONSTANTS`). diff --git a/docs-out/02-language/SCRIPT-BLOCKS.md b/docs-out/02-language/SCRIPT-BLOCKS.md new file mode 100644 index 0000000..24f7d0e --- /dev/null +++ b/docs-out/02-language/SCRIPT-BLOCKS.md @@ -0,0 +1,119 @@ +--- +id: LANG-SCRIPTBLOCK-001 +title: Script blocks +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/lang/scriptblock.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs, src/System.Management.Automation/engine/runtime/ScriptBlockToPowerShell.cs, src/System.Management.Automation/engine/Modules/PSModuleInfo.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/lang/_folder.md] +generated_from: [data/members-tierA.json] +--- + +# Script blocks + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Related: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) (compilation and invocation), [SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md) (`$using:`), [TIERING-POLICY.md](TIERING-POLICY.md). + +## 1. Purpose and scope [SPEC] + +A *script block* (`ScriptBlock`) is the first-class value for a piece of code: the content of `{ … }`, a function body, a script file or a module. It can be stored in variables, passed to cmdlets (`ForEach-Object { }`), invoked with `&`/`.`/`.Invoke()`, bound to a session state (closures, modules) and converted to a .NET delegate. This document covers the `ScriptBlock` type, its creation and sharing rules, invocation entry points, closure binding and `$using:` support. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock@L83-L1085 | partial class | public API: `Create`, `Invoke*`, `GetNewClosure`, `GetPowerShell`, delegates | A | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177 | partial class | cache, `InvokeWithPipeImpl`, compile access | A | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData@L41-L540 | class | shared data (AST, delegates, metadata) | A | +| src/System.Management.Automation/engine/lang/scriptblock.cs::SteppablePipeline@L1095-L1332 | class | step-by-step pipeline from a script block (not analyzed) | A | +| src/System.Management.Automation/engine/runtime/ScriptBlockToPowerShell.cs::ScriptBlockToPowerShellConverter@L210-L952 | class | convert a simple script block to a `PowerShell` object | A | +| src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo@L23-L1554 | class | module object used to bind closures | A | + +## 3. Responsibilities and non-responsibilities + +Value semantics of script blocks: creation, identity, sharing, binding, invocation entry points. Not responsible for compilation internals (T25) or for function/filter definition (command discovery, T32). + +## 4. Key types and contracts [IMPL] + +- `ScriptBlock.Create(string)` parses with a new `Parser` and caches by (file name, text); the internal overload taking an `ExecutionContext` additionally binds the block to the current module session state when the engine is inside a module (src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.Create@L90-L99). +- A `ScriptBlock` object is a thin handle: `_scriptBlockData` (the shared `CompiledScriptBlockData`), a bound session state, and the language mode captured from the creating context. `Clone()` makes a new handle over the **same** data, so compiled delegates are shared between clones and between cached copies (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.TryGetCachedScriptBlock@L583-L601). +- Properties: `Attributes`, `File`, `IsFilter`, `IsConfiguration`, `Module`, `StartPosition`, `LanguageMode`, `OutputType`, `SessionState` (src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock@L83-L1085). +- Invocation: `Invoke(args)` (returns a `Collection`), `InvokeReturnAsIs(args)` (result object as is), `InvokeWithContext(functions, variables, args)` (defines extra functions/variables in a new scope), `InvokeUsingCmdlet`, all funneling into `DoInvoke`/`DoInvokeReturnAsIs` → `InvokeWithPipeImpl` ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.1). `InvokeAsMemberFunction*` serve class methods. `GetDelegate`/`CreateDelegate` convert to a delegate type. + +## 5. Data model + +Handle fields: bound `SessionStateInternal` (or null = use the caller's), `LanguageMode`, debugger-hidden flags; shared data fields: AST, per-clause delegates (optimized and unoptimized), locals tuple types, name → slot map, sequence points, a `Guid` id per data object (new for each parsed script block, not for clones). + +## 6. Control flow + +Creation → (parse) → handle; invocation → bind arguments, choose variant, create scope or push dotted scope, run clause delegate. A closure is created as follows (`GetNewClosure`): make a dynamic module object (`new PSModuleInfo(true)`); `CaptureLocals` copies into the module's session state every variable of the **current scope** (the locals tuple plus the scope's variable table) whose options are `None` (not read-only, constant, private or all-scope, and not the `$null` variable), as new `PSVariable` objects holding the current **values** (a value snapshot, not a reference; parent-scope variables are not captured); `NewBoundScriptBlock` clones the script block with `SessionState` set to the module's session state, so that variables are resolved there when it runs (src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.GetNewClosure@L119-L124, src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo.CaptureLocals@L1343-L1370, src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo.NewBoundScriptBlock@L1245-L1269). + +## 7. Algorithms + +- **Binding to a module/session state**: `PSModuleInfo.Invoke(sb, args)` temporarily sets the script block's session state to the module's and calls `InvokeReturnAsIs`, restoring the old one in a `finally`; `NewBoundScriptBlock` swaps `context.EngineSessionState` under a lock while cloning. +- **`$using:`**: the compiler replaces `$using:name` by `VariableOps.GetUsingValue(locals, key, index, context)`; values are supplied through `$PSBoundParameters.ImplicitUsingParameters` set by the caller that starts the script block elsewhere (jobs, remoting, `ForEach-Object -Parallel`) ([SCOPE-AND-VARIABLES.md](SCOPE-AND-VARIABLES.md) §7.4). `ScriptBlock.GetPowerShell(variables, out usingVariables, …)` extracts the using variables of a block into a dictionary and builds a `PowerShell` object for it (`ScriptBlockToPowerShellConverter`; details of the converter and of its restrictions to simple pipelines are `UNVERIFIED`, `GetSimplePipeline` rules are in [02.3-ast.md](02.3-ast.md) §5 notes). +- **Delegate conversion**: a language conversion from `ScriptBlock` to any delegate type (`ConvertScriptBlockToDelegate`, [CONVERSION-RULES.md](CONVERSION-RULES.md) §7.1 step 8). + +## 8. Observable behavior [SPEC] + +Read from source: two `ScriptBlock.Create` calls with the same text return different handles sharing compiled data; a block created in a module is bound to that module's session state; `$f = { $x }.GetNewClosure()` keeps the value `$x` had when the closure was made (snapshot) while a plain block sees the variable at call time through dynamic scoping. No golden sample: STATIC-ONLY. Tests: `test/powershell/Language/Scripting/` (mapping `UNVERIFIED`). + +## 9. Error handling and edge cases + +`Create` throws `ParseException` for syntax errors; `GetScriptBlock()` on an AST with errors throws; `Invoke` of an empty or missing clause returns nothing; `Clean` is not invokable through `InvokeWithPipeImpl`. Binding a closure to a binary module throws `InvalidOperationOnBinaryModule`. A closure captured inside a loop copies only the current scope's variables, so loop variables from parent scopes are not captured. + +## 10. Concurrency and threading + +Compiled data are shared and published under locks (T25); the script block cache is a `ConcurrentDictionary`; handles are not thread-affine but the session state they bind is. + +## 11. Extension points and customization hooks + +Delegates created from script blocks, `ScriptBlock.Create` for code generation, `InvokeWithContext` to inject functions/variables, `ScriptBlockInvocationEventArgs` for host/job instrumentation (`UNVERIFIED`). + +## 12. Dependencies + +Inbound: functions, scripts, modules, `ForEach-Object`/`Where-Object`, jobs, remoting, event handlers. Outbound: parser, compiler, session state, module system, `PowerShell` API. + +## 13. Performance notes + +Cloning avoids recompilation; `GetNewClosure` allocates a module object and copies variables each time (cost proportional to the scope size). + +## 14. Security considerations (mandatory) + +A script block remembers the language mode of its creator, so passing a constrained block into a full-language session does not elevate it; the transition rules are in `InvokeWithPipeImpl` ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §14). `GetPowerShell` and `$using:` extraction run in an untrusted-input mode unless `isTrustedInput` is passed. + +## 15. Enterprise considerations + +Script block logging uses the shared data's id and text ([02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §14). Missing: no public identity for a block independent of its text. + +## 16. Tests + +Not read (VB-005). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| handle + shared compiled data | `FACT-SRC` | cheap clones, cached compilation; state binding is per handle | +| closures via a dynamic module | `FACT-SRC` | reuses module session state; snapshot semantics; only current-scope variables | +| language mode captured at creation | `FACT-SRC` | policy follows the code, not the caller | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-SB-01** [REPLICATE] (area: NLD-SCP) Make a script block a first-class value with an AST, a bound scope (optional), and a captured language mode (if any); share the parsed/compiled data between clones. Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.TryGetCachedScriptBlock@L583-L601. +- **LSN-SB-02** [REPLICATE] (area: NLD-SCP) `GetNewClosure` snapshots the **current scope's** unrestricted variables by value into a private scope bound to the clone. Evidence: src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo.CaptureLocals@L1343-L1370. +- **LSN-SB-03** [REPLICATE] (area: NLD-SCP) Provide `Invoke`, `InvokeReturnAsIs` (call operator semantics) and `InvokeWithContext` as separate entry points funneling into one invoke routine. Evidence: src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.InvokeWithContext@L412-L443. +- **LSN-SB-04** [ADAPT] (area: NLD-SCP) Implement closures without a module object: a scope object holding the snapshot. Why: same observable behavior; module objects are EXTENDED scope. Evidence: src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.GetNewClosure@L119-L124. +- **LSN-SB-05** [AVOID] (area: NLD-RUN) Omit steppable pipelines and the `PowerShell` object conversion (`GetPowerShell`) in the first versions. Evidence: src/System.Management.Automation/engine/lang/scriptblock.cs::SteppablePipeline@L1095-L1332. DEV: DEV-SB-090 + +## 19. Open questions / UNVERIFIED + +- `ScriptBlockToPowerShellConverter` rules for which blocks convert, parameter and `$using` rewriting (VB-022). +- Behavior of `InvokeWithContext` variable/function injection beyond its signature. +- `SteppablePipeline` and `ScriptBlockInvocationEventArgs` semantics. + +Verification: `scriptblock.cs` public surface (L83-L400, member list L83-L1085), `GetNewClosure`, `PSModuleInfo.CaptureLocals`/`NewBoundScriptBlock`/`Invoke`, `ScriptBlock.TryGetCachedScriptBlock`/`CacheScriptBlock` and `InvokeWithPipeImpl` (earlier notes) were read; the rest of the 1,479-line file is inventory. diff --git a/docs-out/03-runtime/execution-context-and-session-state.md b/docs-out/03-runtime/execution-context-and-session-state.md new file mode 100644 index 0000000..35871d1 --- /dev/null +++ b/docs-out/03-runtime/execution-context-and-session-state.md @@ -0,0 +1,125 @@ +--- +id: RT-CONTEXT-001 +title: Execution context and session state +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/ExecutionContext.cs, src/System.Management.Automation/engine/SessionState.cs, src/System.Management.Automation/engine/SessionStateScopeAPIs.cs, src/System.Management.Automation/engine/SessionStateScope.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/_folder.md] +generated_from: [data/members-tierA.json] +--- + +# Execution context and session state + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Scope and variable rules live in [SCOPE-AND-VARIABLES](../02-language/SCOPE-AND-VARIABLES.md). Compiled code receives the context through `FunctionContext` ([IR-SPEC](../02-language/IR-SPEC.md) §4). + +## 1. Purpose and scope [SPEC] + +`ExecutionContext` is the per-runspace object that holds everything a running script needs: the current session state (scopes, functions, aliases, drives), the language mode, strict mode, the debugger, output pipes, the current exception, the type table and the command discovery object. `SessionState`/`SessionStateInternal` is the tree of scopes and the tables inside them. A new shell needs the same split: a *context* (one per interpreter instance) and a *session state* (one per module or top level). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext@L28-L1679 | class | the per-runspace engine context (`124` members in the scan) | A | +| src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal@L14-L323 | partial class | scope stack: `CurrentScope`, `GlobalScope`, `ScriptScope`, `NewScope`, `RemoveScope` | A | +| src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope@L17-L2012 | class | one scope | A | + +## 3. Responsibilities and non-responsibilities + +Holds state and policy flags; does not execute anything. Command resolution (`CommandDiscovery`) hangs off the context but is owned by T32. + +## 4. Key types and contracts [IMPL] + +- `EngineSessionState` — the session state currently in effect (changes while a module's code runs); `TopLevelSessionState` — the runspace's top-level one (private setter); `SessionState` — the public wrapper of the engine one (src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext@L28-L1679). +- `LanguageMode` (`FullLanguage`, `ConstrainedLanguage`, …): switching to `ConstrainedLanguage` records that the runspace has used it and **invalidates the cached binding rules** (after that the binders encode the language mode in the generated rules); the restore to constrained mode after running trusted code does not invalidate again (src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.LanguageMode@L275-L328). +- `IsStrictVersion(major)`: walks from the current scope to its parents; the first scope with a non-null `StrictModeVersion` decides (`Major >= requested`); a walk does not continue past the module scope; no `Set-StrictMode` → false (src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L105-L110). +- Preference lookup: `GetEnumPreference(path, default, out defaultUsed)` reads the preference variable through the normal variable lookup; a value of type `T` is used directly, a string is parsed case-insensitively, any other value is unwrapped from `PSObject`; invalid values fall back to the default and report that the default was used (src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.GetEnumPreference@L521-L572). +- `CheckStackDepth()` converts `InsufficientExecutionStackException` into `ScriptCallDepthException` (src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.CheckStackDepth@L869-L879). + +## 5. Data model + +Members by role (names from the member scan, behavior `INFERENCE` unless stated): debugger (`Debugger`, `_debuggingMode`, `PSDebugTraceLevel`, `PSDebugTraceStep`), module state (`ModuleBeingProcessed`, `Modules`, `PreviousModuleProcessed`, `AutoLoadingModuleInProgress`), security (`LanguageMode`, `HasEverUsedConstrainedLanguage`, `MarkObjectAsUntrusted`, `AuthorizationManager`), output (`ExternalSuccessOutput`, `ExternalErrorOutput`, `ExternalProgressOutput`, `ShellFunctionErrorOutputPipe`, `Expression*OutputPipe`, `RedirectErrorPipe`, `AppendDollarError`), exception state (`CurrentExceptionBeingHandled`, `PropagateExceptionsToEnclosingStatementBlock`, `QuestionMarkVariableValue`), preference accessors (`DebugPreferenceVariable` … `ConfirmPreferenceVariable`), engine services (`CommandDiscovery`, `TypeTable`, `FormatDBManager`, `AssemblyCache`, `LocationGlobber`, `HelpSystem`, `EngineHostInterface`, `InternalHost`, `Events`). + +## 6. Control flow + +```mermaid +flowchart TD + R[Runspace] --> EC[ExecutionContext] + EC --> TS[TopLevelSessionState: global scope] + EC --> ES[EngineSessionState: current] + ES --> CS[CurrentScope chain] + CS --> VT[variable tables and locals tuples] + EC --> LM[LanguageMode] + EC --> DBG[Debugger and trace level] + EC --> OP[output and error pipes] +``` + +Module code runs with `EngineSessionState` switched to the module's session state; leaving restores it. The scope chain of a module ends at its module scope, which is why strict-mode lookup stops there. + +## 7. Algorithms + +Not applicable beyond §4 (strict mode walk, preference conversion, language-mode switch). + +## 8. Observable behavior [SPEC] + +`$ErrorActionPreference = 'Stop'` is read by error handling through `GetEnumPreference`; an invalid string value is replaced by the default in the engine's view (the visible variable is unchanged, `INFERENCE`). `Set-StrictMode -Version 2` affects code in the scope where it was set and its children until a module boundary. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +`ScriptCallDepthException` on deep recursion; `GetEnumPreference` swallows conversion failures (falls back to default); a null context passed to the static `IsStrictVersion` helper is replaced by the thread-local context. + +## 10. Concurrency and threading + +The context is bound to a runspace and used from its pipeline thread; the language-mode setter takes a lock when invalidating binder caches; the "current context" is found through thread-local storage (`LocalPipeline.GetExecutionContextFromTLS`). + +## 11. Extension points and customization hooks + +Host interface (`EngineHostInterface`), `InitialSessionState`, custom argument completers (`CustomArgumentCompleters`, `NativeArgumentCompleters`), `Events`. + +## 12. Dependencies + +Inbound: everything in the engine (`ExecutionContext` is passed to nearly every runtime helper, see [IR-SPEC](../02-language/IR-SPEC.md) §5). Outbound: session state, providers, debugger, host. + +## 13. Performance notes + +`IsStrictVersion` is called from compiled code paths (strict checks on variable reads, call syntax) and walks scopes each time (`INFERENCE`: cheap when no strict mode is set because the walk reaches the root). + +## 14. Security considerations (mandatory) + +Language mode is a context property; switching it invalidates caches so constrained rules cannot be bypassed by cached full-language binder rules (`FACT-SRC` comment in the setter). `MarkObjectAsUntrusted*` and `IsMarkedAsUntrusted` support the data-trust model of Constrained Language Mode (details: 07-security, planned). + +## 15. Enterprise considerations + +Debug trace levels, host-visible streams and event manager give observability. Missing: no resource accounting per context. + +## 16. Tests + +Not read. + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| one context object passed everywhere | `FACT-SRC` | no global state, easy to test, wide coupling | +| engine session state switched for module code | `FACT-SRC` | modules see their own scope chain | +| language mode in the context plus binder-cache invalidation | `FACT-SRC` | policy enforced at binding time, cached rules must be flushed on change | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-CTX-01** [REPLICATE] (area: NLD-SCP) One interpreter context object holding the current session state, strict mode, preference lookup and error state; pass it to runtime helpers instead of using globals. Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext@L28-L1679. +- **LSN-CTX-02** [REPLICATE] (area: NLD-SCP) Resolve preference variables through the normal variable lookup with a typed fallback. Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.GetEnumPreference@L521-L572. +- **LSN-CTX-03** [REPLICATE] (area: NLD-RUN) Guard recursion with a stack-depth check that raises a dedicated "script call depth" error. Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.CheckStackDepth@L869-L879. +- **LSN-CTX-04** [AVOID] (area: NLD-SEC) Leave out language modes and binder-cache invalidation unless a restricted mode is added later. Evidence: src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.LanguageMode@L275-L328. DEV: DEV-CTX-081 + +## 19. Open questions / UNVERIFIED + +- Only the members named in §4 were read; the rest of the 1,711-line file (output redirection, `AppendDollarError`, assembly cache, startup error reporting) is inventory (names only). `UNVERIFIED` behavior of those members. +- Relationship between `TopLevelSessionState` and nested runspaces. + +Verification: `ExecutionContext.IsStrictVersion`, `GetEnumPreference`, `CheckStackDepth`, `LanguageMode` setter and the `EngineSessionState` declaration were read; the member count comes from `data/members-tierA.json`; the remaining members are listed by name only. diff --git a/docs-out/09-synthesis/deviation-register.md b/docs-out/09-synthesis/deviation-register.md index 37be615..f2db493 100644 --- a/docs-out/09-synthesis/deviation-register.md +++ b/docs-out/09-synthesis/deviation-register.md @@ -21,12 +21,20 @@ All rows are `PROPOSED` (the only status this run may write). `APPROVED`/`REJECT | DEV-ID | status | baseline behavior (CF/GS) | proposed behavior | benefit | compatibility cost | affected CFs | golden samples that would change | verdict locations | |---|---|---|---|---|---|---|---|---| | DEV-AST-020 | PROPOSED | AST nodes carry mutable compiler caches (TupleIndex, RuntimeUsingIndex, TypeDefinitionAst.Type) and implement compile-time assignment protocols (ISupportsAssignment/IAssignableValue). | AST stays purely syntactic; slot indexes and assignment code generation live in compiler side tables keyed by node. | Immutable shareable trees; clean separation of parser and compiler; easier tooling reuse. | None observable by scripts; internal refactor only. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-AST-07 (02-language/02.3-ast.md) | +| DEV-BIND-050 | PROPOSED | Member access goes through COM and ETS-adapter aware binders. | Plain reflection member lookup; no COM binders. | Smaller binder layer. | No COM objects. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-BIND-03 (02-language/BINDER-CATALOG.md) | | DEV-CMD-120 | PROPOSED | Windows runtimes ship `Get-Counter`, `Get-WinEvent`, `New-WinEvent` in the default command set (assembly Microsoft.PowerShell.Commands.Diagnostics). | Do not ship OS-specific diagnostics cmdlets in the default set; offer them as optional modules. | Smaller default attack surface and a portable base command set. | Scripts calling these cmdlets need the optional module installed. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-DIAG-01 (10-command-modules/Microsoft.PowerShell.Commands.Diagnostics.md) | | DEV-CMD-121 | PROPOSED | WSMan client cmdlets and provider (assembly Microsoft.WSMan.Management) are part of the Windows command set. | Omit WSMan commands; remoting transport becomes pluggable behind one session abstraction. | No dependency on a Windows-only protocol stack in the base product. | WSMan-based automation breaks; an optional transport module could restore it. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-WSMAN-01 (10-command-modules/Microsoft.WSMan.Management.md) | | DEV-CMD-122 | PROPOSED | CIM/WMI cmdlets (assembly Microsoft.Management.Infrastructure.CimCmdlets) ship on Windows. | Provide CIM/WMI access only as an optional adapter module. | Keeps the base command set free of Windows management stacks. | Windows management scripts need the optional module. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CIM-01 (10-command-modules/Microsoft.Management.Infrastructure.CimCmdlets.md) | +| DEV-CTX-081 | PROPOSED | ExecutionContext carries language mode and invalidates binder caches on change. | No language modes. | No cache invalidation logic. | No restricted modes. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CTX-04 (03-runtime/execution-context-and-session-state.md) | | DEV-ENG-040 | PROPOSED | Compilation of a top-level script block runs an AMSI scan, admin-block check and suspicious-content hashing. | Compilation is policy-free; an optional policy hook runs before execution. | Smaller engine core; platform-independent. | No AMSI integration or forced logging on suspicious content. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-ENG-07 (02-language/02.5-compiler-and-engine.md) | | DEV-EQV-061 | PROPOSED | Interpreter keeps typed arithmetic/comparison instruction families inherited from the DLR. | Do not port them; numeric semantics come from the evaluator's conversion rules. | Removes dead and defective code. | None observable (instructions unreachable from PowerShell IR). | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-EQV-03 (02-language/EQUIVALENCE-SPEC.md) | +| DEV-FEAT-091 | PROPOSED | workflow/parallel/sequence/inlinescript parse and then fail in semantic checks; from/define/var are reserved. | Reject at parse time. | No dead syntax. | Different error text. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-FEAT-03 (02-language/02.7-language-features.md) | | DEV-GRM-010 | PROPOSED | The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing. | Parsing is pure: directives are recorded in the AST and resolved in a later phase. | Deterministic, side-effect-free parse; safe for tooling and untrusted text. | Errors for a missing assembly appear at a later phase than in PowerShell. | CF-GRM-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PARSER-07 (02-language/02.2-parser.md) | | DEV-LEX-001 | PROPOSED | Tokenizer detects signature-block lookalikes and processes #requires comment lines at parse time. | Omit both from the tokenizer; if wanted, handle pragmas in a dedicated post-tokenize pass. | Keeps the tokenizer a pure text-to-token function; removes security-sensitive fuzzy matching from the hot path. | Scripts using #requires get no version/module check; signed-script trailer detection unavailable. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOK-07 (02-language/02.1-tokenizer.md) | +| DEV-RTOPS-051 | PROPOSED | Classes are emitted with Reflection.Emit and method bodies wrapped by ScriptBlockMemberMethodWrapper. | Classes are interpreter objects, not CLR types. | No Reflection.Emit dependency. | Classes are not usable from other .NET code. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-RTOPS-05 (02-language/RUNTIME-OPERATIONS.md) | +| DEV-SB-090 | PROPOSED | ScriptBlock can become a PowerShell object and a steppable pipeline. | Omitted initially. | Less surface. | No GetPowerShell/GetSteppablePipeline. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SB-05 (02-language/SCRIPT-BLOCKS.md) | +| DEV-SCP-080 | PROPOSED | Values assigned to global variables are marked untrusted in Constrained Language Mode. | No trust marking. | Simpler assignment path. | No Constrained Language Mode. | CF-SCP-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SCOPE-07 (02-language/SCOPE-AND-VARIABLES.md) | | DEV-SEM-030 | PROPOSED | `using module` is resolved at parse time by running Get-Module -ListAvailable in a PowerShell instance. | The directive is recorded; module import happens in a later phase. | Parsing has no side effects or I/O. | Missing-module errors appear later than in PowerShell. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SEM-06 (02-language/02.4-semantic-analysis.md) | | DEV-TIER-060 | PROPOSED | Hot loops are promoted from interpreter to JIT in the background mid-execution via in-place instruction replacement. | No mid-execution promotion; any optimizer runs synchronously at defined points. | Deterministic execution, simpler frames. | Long top-level loops lose the speed-up PowerShell gets. | CF-CMP-002 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TIER-04 (02-language/TIERING-POLICY.md) | +| DEV-TOOLS-100 | PROPOSED | PSParser.Tokenize returns PowerShell 2 style tokens by rewriting the saved token list. | Only the modern token list. | No V2 branches in the parser. | Legacy tools unsupported. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOOLS-03 (02-language/02.8-language-tooling.md) | +| DEV-TYP-070 | PROPOSED | Constrained Language Mode restricts conversion targets inside FigureConversion. | No language mode in the conversion layer; allow-list at the type-system boundary. | Simpler conversion code. | No Constrained Language Mode. | CF-CNV-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CONV-05 (02-language/CONVERSION-RULES.md) | diff --git a/docs-out/CONTRADICTIONS.md b/docs-out/CONTRADICTIONS.md index 3cb0f36..a3443ad 100644 --- a/docs-out/CONTRADICTIONS.md +++ b/docs-out/CONTRADICTIONS.md @@ -14,3 +14,5 @@ Rows are never resolved by silent rewording. Status: `OPEN | RESOLVED`. Final st | CON-008 | `SafeValues.cs` comment: `$OFS` cannot be used, a space is assigned (`src/System.Management.Automation/engine/parser/SafeValues.cs@L579-L581`) | code reads `$OFS` from the session state when a context exists, default space otherwise (`@L546-L556`) | E1 comment vs E1 code | RESOLVED | code beats comments. | docs-out/02-language/02.3-ast.md (§7.2) | | CON-009 | `Compiler.cs` comment near `DefinePowerShellTypes`: class-definition errors should be found at parse time, the compile-time check is a "short term solution" (`src/System.Management.Automation/engine/parser/Compiler.cs@L2717-L2745`) | `TypeDefiner.DefineTypes` reports duplicate/unknown/invalid-member errors only when the root script block is compiled (`src/System.Management.Automation/engine/parser/PSType.cs@L1335-L1431`) | E1 comment vs E1 code | RESOLVED | code is authoritative; documented as a stage (02.4-semantic-analysis.md §6). | docs-out/02-language/02.4-semantic-analysis.md | | CON-010 | study prompt hint: `engine/lang/parserutils.cs` holds parser utilities | file holds runtime operator helpers (`ParserOps`), flow-control exceptions and `InterpreterError` (`src/System.Management.Automation/engine/lang/parserutils.cs@L23-L2010`) | prompt hint vs E1 | RESOLVED | source wins; documented as runtime helpers. | docs-out/02-language/02.5-compiler-and-engine.md | +| CON-011 | `Compiler.BuildHashtable` creates hashtables with `OrdinalIgnoreCase` keys (`src/System.Management.Automation/engine/parser/Compiler.cs@L6533-L6546`) | `HashtableOps.Add` builds the result of `+` with `CurrentCultureIgnoreCase` (`src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs@L1431-L1456`) | E1 code vs E1 code | OPEN | observation recorded in RUNTIME-OPERATIONS §6.3; behavior differs only for culture-sensitive case folding (e.g. Turkish i); needs `pwsh` under such a culture. | docs-out/02-language/RUNTIME-OPERATIONS.md | +| CON-012 | `StringOps.Multiply` comment: negative count "should be a runtime error" (`src/System.Management.Automation/engine/runtime/Operations/StringOps.cs@L27-L35`) | code throws `ArgumentOutOfRangeException` | E1 comment vs E1 code | RESOLVED | code is authoritative; documented in RUNTIME-OPERATIONS §6.3. | docs-out/02-language/RUNTIME-OPERATIONS.md | diff --git a/docs-out/DECISIONS.md b/docs-out/DECISIONS.md index 778e2f9..3055263 100644 --- a/docs-out/DECISIONS.md +++ b/docs-out/DECISIONS.md @@ -20,3 +20,4 @@ | D-016 | `ERROR-CATALOG.md` covers `ParserStrings.resx` only (412 keys); semantic-check keys in that resx are listed in section 4 but explained in T24. | Scope split between T22 and T24. | | D-017 | `AST-CATALOG.md` children column is extracted from `InternalVisit`/`InternalVisitChildren` bodies (receivers of `.InternalVisit(`, loop variables mapped to their collection); the dataset's `ast_typed_members` is not used because it misses tuple and collection children. Statistics: 76 nodes in `ast.cs`, 9 concrete leaf-looking rows (checked: leaves or children visited through `InternalVisitChildren`). | Source reading for `IfStatementAst`, `ScriptBlockAst`, `ErrorStatementAst`, `InvokeMemberExpressionAst`. | | D-018 | T24, T25 and T27 were authored in one batch (inline, no subagents). Drafts use `[[file::Type.Member]]` tokens expanded by `tools/expand_refs.py` from `data/types.json` and `data/members-tierA.json`, so no reference range is typed by hand; unresolved tokens fail the expansion. The T27 documents own CF-CMP-001/002/003; `AlwaysCompile` is documented as unused after `rg` over `engine/`. | Rule 1 (evidence) and rule 9 (generated catalogs). | +| D-019 | T26, T28, T29, T210 and T211 were authored inline (no subagents) in one batch on a branch restarted from the merged `main` (`git checkout -B … origin/main`). No reading notes existed for these areas, so each document states exactly which windows were read in its Verification line and §19; files only partly read are recorded with partial ranges in the ledger (`.work/ledger-proposals/T26_T211.csv`), not as fully read. Coverage for `Binders.cs` (8,015 lines), `LanguagePrimitives.cs` (5,888) and `MiscOps.cs` (3,765) is therefore partial; the gaps are backlog items VB-017..VB-023. | Rule 7 (read windows, no false coverage claims). | diff --git a/docs-out/PROGRESS.md b/docs-out/PROGRESS.md index cfba2cb..936bed1 100644 --- a/docs-out/PROGRESS.md +++ b/docs-out/PROGRESS.md @@ -8,7 +8,7 @@ Run mode: STATIC-ONLY (`ALLOW_STATIC_ONLY=true`). Branch `claude/repo-review-exp | Preflight (section 2) | done | `.work/environment.md`, `.work/pin.json`, DECISIONS D-001..D-011 | | 0 Recon, pinning, tooling | done | datasets in `data/`, tools in `tools/`, registers, glossary, census, dependency graph, source-tree map, 297 folder cards + 40 project cards (generated; annotation overlay pending) | | 1 Repository map | done | layered-architecture, build-system, ci-cd-and-packaging, coding-standards (committed 4a6efaf) | -| 2 Language pipeline | in progress | wave 1: T21 DONE (draft); T22 DONE (draft); T23 DONE (draft); T24 T25 T27 DONE (draft); T26 (binders, runtime operations), T28, T29, T210, T211 and the coordinator documents of 02-language are next | +| 2 Language pipeline | in progress | wave 1: T21 DONE (draft); T22 DONE (draft); T23 DONE (draft); T24 T25 T27 DONE (draft); T26 T28 T29 T210 T211 DONE (draft, partial reading depth: see D-019 and VB-017..VB-023); the coordinator documents of 02-language (pipeline map, stage contracts, call graphs, lifecycle, spec draft, quirks, design space, test corpus) are next | | 3 Runtime | todo | | | 4 Hosting/SDK/remoting/platform/command modules | todo | | | 5 Security | todo | | @@ -27,7 +27,13 @@ T21 tokenizer | T22 parser/grammar/errors | T23 AST | T24 semantic | T25 compile | T22 | `02-language/02.2-parser.md`, `GRAMMAR.ebnf.md`, `ERROR-CATALOG.md` (generated parts by `tools/gen_T22.py`) | draft written; CF-GRM-001, CF-ERR-002, DEV-GRM-010; all checks pass (validate-refs with pin gate bypassed, D-012) | | T23 | `02-language/02.3-ast.md`, `AST-CATALOG.md` (generated by `tools/gen_T23.py`) | draft written; DEV-AST-020, CON-007/008, VB-008..012; all checks pass (validate-refs with pin gate bypassed, D-012) | | T24 | `02-language/02.4-semantic-analysis.md` (check table generated by `tools/gen_T24.py`: 133 sites, 104 error IDs) | draft written; DEV-SEM-030, CON-009; checks pass | -| T25 | `02-language/02.5-compiler-and-engine.md`, `COMPILATION-MAP.md` (`tools/gen_T25.py` + `tools/annotations/T25-compilation-map.json`), `IR-SPEC.md` | draft written; DEV-ENG-040, CON-010, VB-015; checks pass | +| T25 | `02-language/02.5-compiler-and-engine.md`, `COMPILATION-MAP.md` (`tools/gen_T25.py` + `docs-out/tools/annotations/T25-compilation-map.json`), `IR-SPEC.md` | draft written; DEV-ENG-040, CON-010, VB-015; checks pass | | T27 | `02-language/INSTRUCTION-SET.md` (`tools/gen_T27.py`), `TIERING-POLICY.md` (owns CF-CMP-001/002/003), `EQUIVALENCE-SPEC.md` | draft written; DEV-TIER-060, DEV-EQV-061, VB-013/014/016; checks pass | -Next step: T26 (binders, runtime operations), T28 (types, operators, conversions), T29 (scope and variables), T210/T211, then the coordinator documents (pipeline map, stage contracts, lifecycle). Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. +Next step: the coordinator documents of 02-language, then Phase 3 runtime (T31 runspace/pipeline, T32 command discovery, T33 parameter binding and cmdlet infrastructure, T34-T39) and the deeper reads listed in VB-017..VB-023. Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. + +| T26 | `02-language/BINDER-CATALOG.md` (`tools/gen_T26.py` + `tools/annotations/T26-binders.json`), `RUNTIME-OPERATIONS.md` | draft; 30 binders listed (10 read in depth, 20 by names); DEV-BIND-050, DEV-RTOPS-051; CON-011/012; VB-020/021 | +| T28 | `02-language/02.6-types-and-operators.md` (CF-PIP-001), `OPERATOR-TABLE.md` (CF-OP-001, `tools/gen_T28.py`), `CONVERSION-RULES.md` (CF-CNV-001) | draft; DEV-TYP-070; VB-018/019 | +| T29 | `02-language/SCOPE-AND-VARIABLES.md` (CF-SCP-001, `tools/gen_T29.py`), `03-runtime/execution-context-and-session-state.md` | draft; DEV-SCP-080, DEV-CTX-081; VB-017 | +| T210 | `02-language/02.7-language-features.md`, `HOWTO-extend-language.md`, `SCRIPT-BLOCKS.md` | draft; DEV-SB-090, DEV-FEAT-091; VB-022 | +| T211 | `02-language/02.8-language-tooling.md` | draft; DEV-TOOLS-100; VB-023 | diff --git a/docs-out/VERIFICATION-BACKLOG.md b/docs-out/VERIFICATION-BACKLOG.md index f0067e8..afeed45 100644 --- a/docs-out/VERIFICATION-BACKLOG.md +++ b/docs-out/VERIFICATION-BACKLOG.md @@ -20,3 +20,10 @@ Every `UNVERIFIED` claim and every `BLOCKED` item. Under STATIC-ONLY (owner deci | VB-014 | OPEN | LANG-TIERING-001 §16, LANG-EQUIV-001 §16 | No test located that exercises interpreter/JIT parity or loop promotion. | `rg` the xUnit and Pester trees for `CompileSynchronously`, `EnterLoop`, large-loop scripts. | Testability. | | VB-015 | OPEN | LANG-ENGINE-001 §19 | `?.` null-conditional wrapper contains the target expression twice (`Compiler.cs::GetNullConditionalWrappedExpression`); a side-effecting target may be evaluated twice. | Run `(f)?.Length` with a function that logs, in `pwsh`. | Possible semantic bug. | | VB-016 | OPEN | LANG-EQUIV-001 §7.1 | Typed arithmetic/comparison instruction defects (UInt64 cast to Int16, unsigned mapped to signed, `Equal(string)`) are believed unreachable. | Confirm no code path emits typed non-double arithmetic; run unsigned arithmetic in a loop under the interpreter. | Latent interpreter defects. | +| VB-017 | OPEN | LANG-SCOPE-001 §19 | `PSVariable.Value` getter/setter (attribute transformation and validation order, debugger hooks) not read line by line; `Set-Variable -Scope N` indexing not analyzed. | Read `ShellVariable.cs` L15-L754; test with `pwsh`. | Variable semantics detail. | +| VB-018 | OPEN | LANG-CONV-001 §19 | `FigureStaticCreateMethodConversion`, `FigureParseConversion`, `FigureConstructorConversion`, `FigureCastConversion`, `FigurePropertyConversion`, value-based `FigureConversion` and null converters not read in detail. | Read `LanguagePrimitives.cs` L4792-L4860, L5260-L5640. | Conversion detail. | +| VB-019 | OPEN | LANG-TYPES-001 §19 | `PipelineOps.PipelineResult` (captured output to `$null`/object/array) not read. | Read `MiscOps.cs` L832-L860; `$x = 1,2 | %{$_}`, `$x = f` with 0/1/2 outputs in `pwsh`. | Assignment result shapes. | +| VB-020 | OPEN | LANG-BINDERCAT-001 | 20 of 30 binders were annotated from member names and compiler use only. | Read member/index/invoke/set binder bodies in `Binders.cs`. | Binder detail. | +| VB-021 | OPEN | LANG-RTOPS-001 §19 | `PipelineOps.InvokePipeline`, redirections, `.Where()`/`.ForEach()`/member enumeration, `TypeOps`, `SwitchOps`, `ClassOps`, `PipelineOps.Splat` not read. | Read `MiscOps.cs` L29-L890, L2199-L2590, L2623-L3420. | Runtime helper detail. | +| VB-022 | OPEN | LANG-SCRIPTBLOCK-001 §19 | `ScriptBlockToPowerShellConverter` rules and `$using` rewriting not read. | Read `ScriptBlockToPowerShell.cs` (full). | Remoting/jobs support. | +| VB-023 | OPEN | LANG-TOOLING-001 §19 | Completion analysis/completers (Tier C) not read. | Read `CompletionAnalysis.cs`, `CompletionCompleters.cs` if completion is wanted. | Tooling. | diff --git a/docs-out/data/canonical-facts.json b/docs-out/data/canonical-facts.json index 7cbbfea..fb987c1 100644 --- a/docs-out/data/canonical-facts.json +++ b/docs-out/data/canonical-facts.json @@ -56,11 +56,15 @@ { "id": "CF-CNV-001", "subject": "conversion algorithm and converter order", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Conversion algorithm and converter order.** For a source type S and target type T the engine returns the first applicable converter in this order (cached per type pair): (1) a cached entry; (2) if S is the null type: `Nullable<\u2026>` \u2192 null, any reference type \u2192 null, a value type \u2192 no conversion (numeric, `char`, `string`, `bool`, `PSReference`, `switch` and `void` targets have pre-cached null conversions); (3) if T is assignable from S: identity (same type) or assignable; (4) by-ref-like types: none; (5) S is a `PSObject`: none (the caller unwraps first); (6) T is `PSObject`: wrap; (7) under Constrained Language Mode: only targets `object`, `object[]` or the core type list are converted (others are refused, or only audited in WDAC audit mode); (8) **language conversions**: `Nullable` through U, `void`, `bool` (from `IList`, enum, value type, class), `string`, arrays (related arrays element-wise, unrelated arrays per element, enumerable to array, scalar to one-element array), `Array`, `Hashtable` from `IDictionary`, `PSReference`, `XmlDocument`, `StringCollection`, `ScriptBlock` to delegate, `PSCustomObject`, integer to enum, `PSMethod` to delegate; (9) a static `Parse` method on T; (10) a static `Create` method; (11) a constructor of T taking S; (12) a cast operator (implicit, then explicit) on S or T; (13) `IConvertible` (numeric sources through the string-convertible path, other `IConvertible` sources via `Convert`); (14) for an `IDictionary` source: T's no-argument constructor then property assignment from the keys; (15) an `IEnumerable` constructor; (16) property-based conversion; if S or T may have a `TypeConverter`/`PSTypeConverter`, or a value-dependent converter was found, the result is wrapped to try the custom/value-dependent converter first and the found converter second; (17) otherwise no conversion (rank `None`, a failing converter that raises `PSInvalidCastException`). Numeric-to-numeric, number-to-string, number-to-bool and null-to-number conversions are pre-cached with fixed ranks.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828", + "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureLanguageConversion@L4966-L5120" + ], "owner": "02-language/CONVERSION-RULES.md", - "status": "open" + "status": "open", + "_task": "T28" }, { "id": "CF-DSC-001", @@ -178,11 +182,15 @@ { "id": "CF-OP-001", "subject": "operator precedence and associativity", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Operator precedence and associativity.** Binary operators are parsed by precedence climbing and **all are left-associative**; from lowest to highest precedence the levels are: logical (`-and`, `-or`, `-xor`) = 1; bitwise (`-band`, `-bor`, `-bxor`) = 2; comparison group (all `-eq`/`-ne`/`-gt`/`-ge`/`-lt`/`-le`, `-like`, `-match`, `-replace`, `-contains`, `-in` families with their `i`/`c` variants, `-is`, `-isnot`, `-as`, `-split`, `-join`, `-shl`, `-shr`) = 5; null-coalescing `??` = 7; additive (`+`, `-`) = 9; multiplicative (`*`, `/`, `%`) = 10; format `-f` = 12; range `..` = 13. The comma (array-literal) operator binds tighter than every binary operator; the unary operators, casts and `++`/`--` bind tighter than the comma; the ternary `?:` (right-nested) and assignment (`=`, `+=`, `-=`, `*=`, `/=`, `%=`, `??=`, parsed at statement level with a statement on the right) bind looser than all binary operators; `&&` and `||` chain pipelines and are not expression operators. Operators spelled with a leading dash are recognized as operators only when the tokenizer is in expression mode.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950", + "src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752" + ], "owner": "02-language/OPERATOR-TABLE.md", - "status": "open" + "status": "open", + "_task": "T28" }, { "id": "CF-PIN-001", @@ -196,20 +204,32 @@ { "id": "CF-PIP-001", "subject": "output unrolling and enumeration rules", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Output unrolling and enumeration rules.** A value is *enumerated* (treated as a sequence) when, after unwrapping `PSObject`: it is `AutomationNull` (empty sequence), an array, a `DataTable` (its rows), a COM enumerable, any other `IEnumerable` that is not a string, `IDictionary` or `XmlNode` (the generic `IEnumerable` enumerator is preferred), or an `IEnumerator`; `null`, strings, dictionaries (hashtables), `XmlNode` and all other objects are **not** enumerated and are single values. When a statement writes its value to the output pipe: `AutomationNull` writes nothing; a non-enumerable value is written as one object (mutable value types are copied first); an enumerable value is **unrolled exactly one level**: each element is written separately (elements that are themselves collections stay whole), and an enumerator is disposed afterwards if the engine created it.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L698", + "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipeWriterBinder@L841-L926", + "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::EnumerableOps.WriteEnumerableToPipe@L3623-L3640" + ], "owner": "02-language/02.6-types-and-operators.md", - "status": "open" + "status": "open", + "_task": "T28" }, { "id": "CF-SCP-001", "subject": "scope, variable lookup and assignment rules", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Scope, lookup and assignment rules.** A variable name is parsed into an optional scope qualifier (`global:`, `local:`, `script:`, `private:`) or a provider drive (`name:`), and an unqualified name. **Reading:** an unqualified name is searched in the current scope and then in each parent scope up to the global scope, and the first match wins; a variable marked `Private` is skipped when it is found in any scope other than the current one; `global:`, `local:`/`private:` and `script:` search exactly one scope (the global scope, the current scope, the script scope respectively); inside one scope the compiled locals tuple (and any dot-sourced tuples) is checked before the variable table; a drive-qualified name is read through the provider. A name that is not found yields `$null`, or a `VariableIsUndefined` error under strict mode (version 2: always; version 1: except inside expandable strings). **Writing:** an unqualified or `local:`/`private:` assignment always targets the current scope (a parent's variable of the same name is never modified; a new local variable is created if the current scope has none), `script:` and `global:` target the script and global scope; assigning to an existing variable in the target scope keeps its object and replaces its value after running its attributes (transformation, then validation), and a `ReadOnly` or `Constant` variable cannot be written (`VariableNotWritable`; `-Force` can overwrite `ReadOnly` but never `Constant`). **New scopes:** every function or script block call (not dot-sourcing) creates a child scope whose parent is the caller's scope; the script scope is the nearest enclosing scope created for a script file, or the global scope; variables marked `AllScope` are shared by reference with every new scope. **Dot-sourcing** creates no scope: the callee's locals tuple is pushed onto the current scope's `DottedScopes` for the duration of the call.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297", + "src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.SetVariable@L401-L518", + "src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L19-L120", + "src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal.NewScope@L227-L240", + "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L117-L143" + ], "owner": "02-language/SCOPE-AND-VARIABLES.md", - "status": "open" + "status": "open", + "_task": "T29" }, { "id": "CF-SEC-001", diff --git a/docs-out/data/claims/T26.jsonl b/docs-out/data/claims/T26.jsonl new file mode 100644 index 0000000..414e9d0 --- /dev/null +++ b/docs-out/data/claims/T26.jsonl @@ -0,0 +1,6 @@ +{"id": "CLM-RTOPS-0001", "subject_key": "erroraction.default", "predicate": "ErrorActionPreference default in statement error handling", "value": "Continue", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps@L1460-L2197"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0002", "subject_key": "erroraction.throw_statement", "predicate": "exception from a throw statement without traps", "value": "rethrown", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps@L1460-L2197"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0003", "subject_key": "trap.continue", "predicate": "continue inside a trap", "value": "SilentlyContinue", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps@L1460-L2197"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0004", "subject_key": "loopflow.empty_label", "predicate": "empty break/continue label", "value": "matches any loop", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/lang/parserutils.cs::LoopFlowException@L36-L67"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0005", "subject_key": "hashtable.literal.duplicate", "predicate": "duplicate or null key in hashtable literal", "value": "runtime error", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::HashtableOps@L1402-L1458"], "doc": "LANG-RTOPS-001", "cf": null} +{"id": "CLM-RTOPS-0006", "subject_key": "binder.count", "predicate": "binder classes in dataset", "value": "30", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422"], "doc": "LANG-RTOPS-001", "cf": null} diff --git a/docs-out/data/claims/T28.jsonl b/docs-out/data/claims/T28.jsonl new file mode 100644 index 0000000..c8c48c6 --- /dev/null +++ b/docs-out/data/claims/T28.jsonl @@ -0,0 +1,16 @@ +{"id": "CLM-TYP-0001", "subject_key": "enumeration.string", "predicate": "string is enumerated by PSEnumerableBinder", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L698"], "doc": "LANG-TYPES-001", "cf": "CF-PIP-001"} +{"id": "CLM-TYP-0002", "subject_key": "enumeration.dictionary", "predicate": "IDictionary is enumerated", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L698"], "doc": "LANG-TYPES-001", "cf": "CF-PIP-001"} +{"id": "CLM-TYP-0003", "subject_key": "output.unroll_depth", "predicate": "pipe write unrolls enumerables by", "value": "one level", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::EnumerableOps.WriteEnumerableToPipe@L3623-L3640"], "doc": "LANG-TYPES-001", "cf": "CF-PIP-001"} +{"id": "CLM-TYP-0004", "subject_key": "arith.int_overflow", "predicate": "int add/sub/mul overflow result type", "value": "double", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0005", "subject_key": "arith.int_divide", "predicate": "int/int result", "value": "int if exact else double", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0006", "subject_key": "arith.zero_divide", "predicate": "division by zero", "value": "RuntimeException", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0007", "subject_key": "truthiness.string", "predicate": "string truthiness", "value": "non-empty", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1018"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0008", "subject_key": "truthiness.list", "predicate": "single-element list truthiness", "value": "truthiness of the element (list element: non-empty)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1018"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0009", "subject_key": "pipelineresult.bool", "predicate": "captured pipeline result to bool", "value": "0:false, 1:element as bool, >1:true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSPipelineResultToBoolBinder@L1119-L1178"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0010", "subject_key": "destructuring.shape", "predicate": "array destructuring with more values than targets", "value": "last target receives the remaining values as a slice", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSArrayAssignmentRHSBinder@L935-L1042"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0011", "subject_key": "convert.double_format", "predicate": "double to string format", "value": "G15 (invariant)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives@L308-L309"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0012", "subject_key": "convert.string_to_number", "predicate": "string to number uses", "value": "Parser.ScanNumber", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.TryScanNumber@L2882-L2900"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0013", "subject_key": "convert.constrained_gate", "predicate": "constrained language conversion targets", "value": "object, object[] or CoreTypes only", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828"], "doc": "LANG-TYPES-001", "cf": "CF-CNV-001"} +{"id": "CLM-TYP-0014", "subject_key": "operator.precedence.levels", "predicate": "binary precedence levels", "value": "logical1 bitwise2 comparison5 coalesce7 add9 multiply10 format12 range13", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752"], "doc": "LANG-TYPES-001", "cf": "CF-OP-001"} +{"id": "CLM-TYP-0015", "subject_key": "repeat.restricted_cap", "predicate": "string/array repetition cap in RestrictedLanguage", "value": "1024", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/StringOps.cs::StringOps@L15-L107"], "doc": "LANG-TYPES-001", "cf": null} +{"id": "CLM-TYP-0016", "subject_key": "hashtable.add.comparer", "predicate": "hashtable + hashtable comparer", "value": "CurrentCultureIgnoreCase", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::HashtableOps@L1402-L1458"], "doc": "LANG-TYPES-001", "cf": null} diff --git a/docs-out/data/claims/T29.jsonl b/docs-out/data/claims/T29.jsonl new file mode 100644 index 0000000..69da0d8 --- /dev/null +++ b/docs-out/data/claims/T29.jsonl @@ -0,0 +1,11 @@ +{"id": "CLM-SCP-0001", "subject_key": "scope.read.unqualified", "predicate": "unqualified variable read searches", "value": "current scope then parents; first match wins", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScopedItemSearcher.cs::ScopedItemSearcher@L15-L239"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0002", "subject_key": "scope.read.private_skip", "predicate": "private variable found outside the current scope", "value": "skipped", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0003", "subject_key": "scope.write.unqualified", "predicate": "unqualified assignment targets", "value": "current scope (forced local)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L19-L120"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0004", "subject_key": "scope.allscope.copy", "predicate": "AllScope variables in new scopes", "value": "same object copied by reference from the parent", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.AddSessionStateScopeDefaultVariables@L1756-L1782"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0005", "subject_key": "scope.global.defaults", "predicate": "default variables of the global scope", "value": "null, false, true (AllScope)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.AddSessionStateScopeDefaultVariables@L1756-L1782"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0006", "subject_key": "scope.automatic.slots", "predicate": "fixed automatic variable slots", "value": "9 (Underbar..PSCommandPath)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SpecialVariables.cs::AutomaticVariable@L401-L413"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0007", "subject_key": "scope.preference.slots", "predicate": "preference variable slots", "value": "8 (Debug..Progress), indexes 9-16", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/SpecialVariables.cs::PreferenceVariable@L415-L425"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0008", "subject_key": "scope.strict.walk", "predicate": "strict mode lookup stops at", "value": "module scope", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L117-L143"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0009", "subject_key": "scope.strict.version1_exception", "predicate": "strict version 1 undefined variable error", "value": "not raised inside expandable strings", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps@L15-L326"], "doc": "LANG-SCOPE-001", "cf": "CF-SCP-001"} +{"id": "CLM-SCP-0010", "subject_key": "variablepath.drive_rule", "predicate": "path with a colon after index 0 and no scope prefix", "value": "drive qualified", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/VariablePath.cs::VariablePath@L29-L313"], "doc": "LANG-SCOPE-001", "cf": null} +{"id": "CLM-SCP-0011", "subject_key": "closure.captures", "predicate": "GetNewClosure copies", "value": "current scope variables with Options None, by value", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo.CaptureLocals@L1343-L1370"], "doc": "LANG-SCOPE-001", "cf": null} diff --git a/docs-out/data/lessons.json b/docs-out/data/lessons.json index b774b64..d61cced 100644 --- a/docs-out/data/lessons.json +++ b/docs-out/data/lessons.json @@ -153,6 +153,50 @@ "file": "01-repository/coding-standards.md", "line": 111 }, + { + "id": "LSN-CTX-01", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "One interpreter context object holding the current session state, strict mode, preference lookup and error state; pass it to runtime helpers instead of using globals.", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext@L28-L1679.", + "dev": null, + "doc": "RT-CONTEXT-001", + "file": "03-runtime/execution-context-and-session-state.md", + "line": 115 + }, + { + "id": "LSN-CTX-02", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Resolve preference variables through the normal variable lookup with a typed fallback.", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.GetEnumPreference@L521-L572.", + "dev": null, + "doc": "RT-CONTEXT-001", + "file": "03-runtime/execution-context-and-session-state.md", + "line": 116 + }, + { + "id": "LSN-CTX-03", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Guard recursion with a stack-depth check that raises a dedicated \"script call depth\" error.", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.CheckStackDepth@L869-L879.", + "dev": null, + "doc": "RT-CONTEXT-001", + "file": "03-runtime/execution-context-and-session-state.md", + "line": 117 + }, + { + "id": "LSN-CTX-04", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Leave out language modes and binder-cache invalidation unless a restricted mode is added later.", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.LanguageMode@L275-L328.", + "dev": "DEV-CTX-081", + "doc": "RT-CONTEXT-001", + "file": "03-runtime/execution-context-and-session-state.md", + "line": 118 + }, { "id": "LSN-TOK-01", "verdict": "REPLICATE", @@ -538,6 +582,226 @@ "file": "02-language/02.5-compiler-and-engine.md", "line": 189 }, + { + "id": "LSN-TYPES-01", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Implement the enumeration rule and one-level unrolling exactly as in \u00a77.1, including `AutomationNull`, strings and dictionaries as non-enumerated.", + "evidence": "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder.FallbackConvert@L580-L693.", + "dev": null, + "doc": "LANG-TYPES-001", + "file": "02-language/02.6-types-and-operators.md", + "line": 125 + }, + { + "id": "LSN-TYPES-02", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Arithmetic: wider-type promotion, `int` overflow to `double`, exact `int/int` stays `int`, zero division errors.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/NumericOps.cs::IntOps@L15-L129.", + "dev": null, + "doc": "LANG-TYPES-001", + "file": "02-language/02.6-types-and-operators.md", + "line": 126 + }, + { + "id": "LSN-TYPES-03", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Collections on the left of comparison, `-like` and `-match` filter; scalar `-match` sets `$Matches`.", + "evidence": "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.BinaryComparisonCommon@L3378-L3419.", + "dev": null, + "doc": "LANG-TYPES-001", + "file": "02-language/02.6-types-and-operators.md", + "line": 127 + }, + { + "id": "LSN-TYPES-04", + "verdict": "ADAPT", + "area": "NLD-OP", + "text": "Implement `op_*` user-defined operators only for MicroteamShell's own types. Why: avoids reflection over arbitrary .NET types; same observable behavior for supported types.", + "evidence": "src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.CallImplicitOp@L2290-L2311.", + "dev": null, + "doc": "LANG-TYPES-001", + "file": "02-language/02.6-types-and-operators.md", + "line": 128 + }, + { + "id": "LSN-TYPES-05", + "verdict": "ADAPT", + "area": "NLD-TYP", + "text": "Add a regex match timeout and size limits for string/array repetition in every mode. Why: PowerShell caps repetition only in restricted mode and no regex timeout was found (`UNVERIFIED`); a personal shell should fail safely.", + "evidence": "src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1624.", + "dev": null, + "doc": "LANG-TYPES-001", + "file": "02-language/02.6-types-and-operators.md", + "line": 129 + }, + { + "id": "LSN-FEAT-01", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Build the CORE rows first, in the order tokenizer, parser, AST, evaluator, scopes, operators, commands/pipeline, flow control, errors.", + "evidence": "src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8187.", + "dev": null, + "doc": "LANG-FEATURES-001", + "file": "02-language/02.7-language-features.md", + "line": 133 + }, + { + "id": "LSN-FEAT-02", + "verdict": "ADAPT", + "area": "NLD-GRM", + "text": "Add the EXTENDED rows (classes, modules, providers) only after the CORE rows pass the corpus; treat DSC and workflow as absent. Why: same language core with far less surface.", + "evidence": "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ConfigurationStatementRule@L2893-L3322.", + "dev": null, + "doc": "LANG-FEATURES-001", + "file": "02-language/02.7-language-features.md", + "line": 134 + }, + { + "id": "LSN-FEAT-03", + "verdict": "AVOID", + "area": "NLD-GRM", + "text": "Do not implement `workflow`/`parallel`/`sequence`/`inlinescript` or the reserved keywords `from`, `define`, `var`; reject them with the same kind of parse error.", + "evidence": "src/System.Management.Automation/engine/parser/Parser.cs::Parser.BlockStatementRule@L2341-L2359.", + "dev": "DEV-FEAT-091", + "doc": "LANG-FEATURES-001", + "file": "02-language/02.7-language-features.md", + "line": 135 + }, + { + "id": "LSN-TOOLS-01", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Expose `Parse(text)` returning AST, tokens (with comments) and all errors, with error nodes for broken input, from day one.", + "evidence": "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseInput@L131-L134.", + "dev": null, + "doc": "LANG-TOOLING-001", + "file": "02-language/02.8-language-tooling.md", + "line": 117 + }, + { + "id": "LSN-TOOLS-02", + "verdict": "REPLICATE", + "area": "NLD-TOOL", + "text": "Provide `Find/FindAll` over the AST and a pure \"literal data\" evaluator for manifests and configuration.", + "evidence": "src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420.", + "dev": null, + "doc": "LANG-TOOLING-001", + "file": "02-language/02.8-language-tooling.md", + "line": 118 + }, + { + "id": "LSN-TOOLS-03", + "verdict": "AVOID", + "area": "NLD-TOOL", + "text": "Do not build the PowerShell 2 compatible token API or the V2 token rewriting.", + "evidence": "src/System.Management.Automation/engine/lang/interface/PSParser.cs::PSParser@L64-L193.", + "dev": "DEV-TOOLS-100", + "doc": "LANG-TOOLING-001", + "file": "02-language/02.8-language-tooling.md", + "line": 119 + }, + { + "id": "LSN-TOOLS-04", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "If completion is added, keep type inference side-effect free (no command execution, no CIM lookups). Why: same completion quality for the shell's own types without running code.", + "evidence": "src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs::AstTypeInference@L40-L110.", + "dev": null, + "doc": "LANG-TOOLING-001", + "file": "02-language/02.8-language-tooling.md", + "line": 120 + }, + { + "id": "LSN-BIND-01", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Replace DLR binders by explicit dispatch functions per operation (`Add(left, right)`, `Convert(value, type)`, `GetMember(target, name)`, `Index(target, args)`) with a small per-site cache only if profiling shows a need. Why: the DLR is a mechanism, the observable behavior is the rule set.", + "evidence": "`src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`.", + "dev": null, + "doc": "LANG-BINDERCAT-001", + "file": "02-language/BINDER-CATALOG.md", + "line": 78 + }, + { + "id": "LSN-BIND-02", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Keep the dedicated helper operations (enumerate, to-object-array, write-to-pipe, array-assignment shaping, pipeline-result-to-bool); they encode language rules.", + "evidence": "`src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder@L518-L780`, `PSPipeWriterBinder@L841-L926`, `PSArrayAssignmentRHSBinder@L935-L1042`, `PSPipelineResultToBoolBinder@L1119-L1178`.", + "dev": null, + "doc": "LANG-BINDERCAT-001", + "file": "02-language/BINDER-CATALOG.md", + "line": 79 + }, + { + "id": "LSN-BIND-03", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not port COM binders and `PSGetMemberBinder`'s COM/ETS-adapter paths in the first versions. Replace with a plain reflection member lookup.", + "evidence": "`src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComBinder@L1-L400`.", + "dev": "DEV-BIND-050", + "doc": "LANG-BINDERCAT-001", + "file": "02-language/BINDER-CATALOG.md", + "line": 80 + }, + { + "id": "LSN-CONV-01", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Use the search order of \u00a77.1 with a per-(source,target) cache and an explicit rank for each outcome.", + "evidence": "`src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828`.", + "dev": null, + "doc": "LANG-CONV-001", + "file": "02-language/CONVERSION-RULES.md", + "line": 190 + }, + { + "id": "LSN-CONV-02", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Convert strings to numbers with the same scanner as the tokenizer (hex, binary, suffixes, multipliers) and numbers to strings with invariant formatting (`G15`/`G7`, literal token text preserved).", + "evidence": "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.TryScanNumber@L2882-L2906, src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.ConvertNumericToString@L3322-L3363.", + "dev": null, + "doc": "LANG-CONV-001", + "file": "02-language/CONVERSION-RULES.md", + "line": 191 + }, + { + "id": "LSN-CONV-03", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Implement `IsTrue` exactly as in \u00a77.3, including the one-element-list rule.", + "evidence": "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.IsTrue@L980-L1017.", + "dev": null, + "doc": "LANG-CONV-001", + "file": "02-language/CONVERSION-RULES.md", + "line": 192 + }, + { + "id": "LSN-CONV-04", + "verdict": "ADAPT", + "area": "NLD-TYP", + "text": "Support the language conversions (\u00a77.1 step 8) and `Parse`/constructor/cast mechanisms for MicroteamShell's own types first; add reflection-based discovery of .NET `Parse`/`Create`/constructors only for the .NET types the shell exposes. Why: same observable casts for supported types without reflection over arbitrary assemblies.", + "evidence": "src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureLanguageConversion@L4966-L5154.", + "dev": null, + "doc": "LANG-CONV-001", + "file": "02-language/CONVERSION-RULES.md", + "line": 193 + }, + { + "id": "LSN-CONV-05", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Do not copy the Constrained-Language gate or the WDAC audit hook in the conversion layer; use an allow-list of target types at the shell's type-system boundary if a restricted mode is wanted.", + "evidence": "`src/System.Management.Automation/engine/LanguagePrimitives.cs::LanguagePrimitives.FigureConversion@L5641-L5828`.", + "dev": "DEV-TYP-070", + "doc": "LANG-CONV-001", + "file": "02-language/CONVERSION-RULES.md", + "line": 194 + }, { "id": "LSN-EQV-01", "verdict": "REPLICATE", @@ -571,6 +835,39 @@ "file": "02-language/EQUIVALENCE-SPEC.md", "line": 126 }, + { + "id": "LSN-HOWTO-01", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Keep one declaration per operator/keyword (spelling, kind, precedence, flags, evaluator) and generate the tables, instead of parallel arrays kept in sync by assertions. Why: same behavior, less drift.", + "evidence": "src/System.Management.Automation/engine/parser/token.cs::TokenTraits@L757-L1211.", + "dev": null, + "doc": "LANG-HOWTO-001", + "file": "02-language/HOWTO-extend-language.md", + "line": 143 + }, + { + "id": "LSN-HOWTO-02", + "verdict": "ADAPT", + "area": "NLD-TOOL", + "text": "Give every AST node one visitor interface with a default method; add a test that visits all nodes of a corpus with each visitor. Why: PowerShell's two-generation visitor scheme is only for compatibility.", + "evidence": "src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235.", + "dev": null, + "doc": "LANG-HOWTO-001", + "file": "02-language/HOWTO-extend-language.md", + "line": 144 + }, + { + "id": "LSN-HOWTO-03", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Classify each new construct for restricted evaluation at the time it is added.", + "evidence": "src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454.", + "dev": null, + "doc": "LANG-HOWTO-001", + "file": "02-language/HOWTO-extend-language.md", + "line": 145 + }, { "id": "LSN-IR-01", "verdict": "ADAPT", @@ -626,6 +923,226 @@ "file": "02-language/IR-SPEC.md", "line": 186 }, + { + "id": "LSN-OP-01", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Use the precedence levels and left associativity above; keep the comma tighter than binary operators and put ternary and assignment below.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950`.", + "dev": null, + "doc": "LANG-OPTABLE-001", + "file": "02-language/OPERATOR-TABLE.md", + "line": 134 + }, + { + "id": "LSN-OP-02", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Choose the operation by the left operand's type and convert the right operand to it (CONVERSION-RULES); keep the `i`/`c` case variants and the array-filtering behavior of comparison operators.", + "evidence": "`src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`.", + "dev": null, + "doc": "LANG-OPTABLE-001", + "file": "02-language/OPERATOR-TABLE.md", + "line": 135 + }, + { + "id": "LSN-OP-03", + "verdict": "ADAPT", + "area": "NLD-OP", + "text": "Implement operators as ordinary functions with an explicit dispatch table instead of dynamic binding sites. Why: same observable results; no DLR.", + "evidence": "`src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.FallbackBinaryOperation@L2179-L2240`.", + "dev": null, + "doc": "LANG-OPTABLE-001", + "file": "02-language/OPERATOR-TABLE.md", + "line": 136 + }, + { + "id": "LSN-RTOPS-01", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Implement the seven-step error-action algorithm of \u00a76.1 as one function used by the statement runner.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::ExceptionHandlingOps.CheckActionPreference@L1699-L1788.", + "dev": null, + "doc": "LANG-RTOPS-001", + "file": "02-language/RUNTIME-OPERATIONS.md", + "line": 182 + }, + { + "id": "LSN-RTOPS-02", + "verdict": "REPLICATE", + "area": "NLD-PIP", + "text": "Use label-carrying break/continue signals where an empty label matches any loop.", + "evidence": "src/System.Management.Automation/engine/lang/parserutils.cs::LoopFlowException@L36-L67.", + "dev": null, + "doc": "LANG-RTOPS-001", + "file": "02-language/RUNTIME-OPERATIONS.md", + "line": 183 + }, + { + "id": "LSN-RTOPS-03", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Hashtable literal construction rejects null and duplicate keys; `+` of hashtables creates a new table and throws on a shared key.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs::HashtableOps@L1402-L1458.", + "dev": null, + "doc": "LANG-RTOPS-001", + "file": "02-language/RUNTIME-OPERATIONS.md", + "line": 184 + }, + { + "id": "LSN-RTOPS-04", + "verdict": "ADAPT", + "area": "NLD-OP", + "text": "Cap string and array repetition in all modes (not only data sections). Why: PowerShell caps only in restricted mode; a personal shell should not be able to exhaust memory by accident.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/StringOps.cs::StringOps@L15-L107, src/System.Management.Automation/engine/runtime/Operations/ArrayOps.cs::ArrayOps@L13-L301.", + "dev": null, + "doc": "LANG-RTOPS-001", + "file": "02-language/RUNTIME-OPERATIONS.md", + "line": 185 + }, + { + "id": "LSN-RTOPS-05", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not port the class-method wrapper machinery (`ScriptBlockMemberMethodWrapper`, `SessionStateKeeper`, Reflection.Emit class definition) in the first versions; classes are EXTENDED scope.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/ClassOps.cs::ScriptBlockMemberMethodWrapper@L98-L269.", + "dev": "DEV-RTOPS-051", + "doc": "LANG-RTOPS-001", + "file": "02-language/RUNTIME-OPERATIONS.md", + "line": 186 + }, + { + "id": "LSN-SCOPE-01", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Dynamic scoping for reads (walk to parents, skip `private`), local-only writes, explicit `script:`/`global:`/`local:`/`private:` qualifiers.", + "evidence": "src/System.Management.Automation/engine/ScopedItemSearcher.cs::VariableScopeItemSearcher@L244-L297, src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121.", + "dev": null, + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 227 + }, + { + "id": "LSN-SCOPE-02", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Each function/script-block call creates a child scope; dot-sourcing runs in the caller's scope; a script file is a script scope.", + "evidence": "src/System.Management.Automation/engine/SessionStateScopeAPIs.cs::SessionStateInternal.NewScope@L227-L243.", + "dev": null, + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 228 + }, + { + "id": "LSN-SCOPE-03", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Type constraints on assignment (`[int]$x = \u2026`) attach transformation/validation attributes to the variable and replace the previous ones on re-declaration.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121.", + "dev": null, + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 229 + }, + { + "id": "LSN-SCOPE-04", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Support `ReadOnly`, `Constant`, `Private`, `AllScope`, with the write rules of \u00a77.1.", + "evidence": "src/System.Management.Automation/engine/ShellVariable.cs::ScopedItemOptions@L882-L915.", + "dev": null, + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 230 + }, + { + "id": "LSN-SCOPE-05", + "verdict": "ADAPT", + "area": "NLD-SCP", + "text": "Keep one variable store per scope (no tuple slots) in the first implementation, but keep the lookup API narrow so slots can be added. Why: same observable behavior; slots are an optimization.", + "evidence": "src/System.Management.Automation/engine/SessionStateScope.cs::SessionStateScope.TryGetLocalVariableFromTuple@L680-L692.", + "dev": null, + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 231 + }, + { + "id": "LSN-SCOPE-06", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Strict-mode version is found by walking scopes, stopping at the module scope; version 0 means off.", + "evidence": "src/System.Management.Automation/engine/ExecutionContext.cs::ExecutionContext.IsStrictVersion@L105-L110.", + "dev": null, + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 232 + }, + { + "id": "LSN-SCOPE-07", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Do not implement the untrusted-value marking tied to Constrained Language Mode unless a language-mode feature is adopted.", + "evidence": "src/System.Management.Automation/engine/runtime/Operations/VariableOps.cs::VariableOps.SetVariableValue@L17-L121.", + "dev": "DEV-SCP-080", + "doc": "LANG-SCOPE-001", + "file": "02-language/SCOPE-AND-VARIABLES.md", + "line": 233 + }, + { + "id": "LSN-SB-01", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Make a script block a first-class value with an AST, a bound scope (optional), and a captured language mode (if any); share the parsed/compiled data between clones.", + "evidence": "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.TryGetCachedScriptBlock@L583-L601.", + "dev": null, + "doc": "LANG-SCRIPTBLOCK-001", + "file": "02-language/SCRIPT-BLOCKS.md", + "line": 107 + }, + { + "id": "LSN-SB-02", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "`GetNewClosure` snapshots the **current scope's** unrestricted variables by value into a private scope bound to the clone.", + "evidence": "src/System.Management.Automation/engine/Modules/PSModuleInfo.cs::PSModuleInfo.CaptureLocals@L1343-L1370.", + "dev": null, + "doc": "LANG-SCRIPTBLOCK-001", + "file": "02-language/SCRIPT-BLOCKS.md", + "line": 108 + }, + { + "id": "LSN-SB-03", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Provide `Invoke`, `InvokeReturnAsIs` (call operator semantics) and `InvokeWithContext` as separate entry points funneling into one invoke routine.", + "evidence": "src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.InvokeWithContext@L412-L443.", + "dev": null, + "doc": "LANG-SCRIPTBLOCK-001", + "file": "02-language/SCRIPT-BLOCKS.md", + "line": 109 + }, + { + "id": "LSN-SB-04", + "verdict": "ADAPT", + "area": "NLD-SCP", + "text": "Implement closures without a module object: a scope object holding the snapshot. Why: same observable behavior; module objects are EXTENDED scope.", + "evidence": "src/System.Management.Automation/engine/lang/scriptblock.cs::ScriptBlock.GetNewClosure@L119-L124.", + "dev": null, + "doc": "LANG-SCRIPTBLOCK-001", + "file": "02-language/SCRIPT-BLOCKS.md", + "line": 110 + }, + { + "id": "LSN-SB-05", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Omit steppable pipelines and the `PowerShell` object conversion (`GetPowerShell`) in the first versions.", + "evidence": "src/System.Management.Automation/engine/lang/scriptblock.cs::SteppablePipeline@L1095-L1332.", + "dev": "DEV-SB-090", + "doc": "LANG-SCRIPTBLOCK-001", + "file": "02-language/SCRIPT-BLOCKS.md", + "line": 111 + }, { "id": "LSN-TIER-01", "verdict": "REPLICATE", diff --git a/docs-out/data/numbers-extra.json b/docs-out/data/numbers-extra.json index 93a0eaf..ea2b2b7 100644 --- a/docs-out/data/numbers-extra.json +++ b/docs-out/data/numbers-extra.json @@ -117,5 +117,68 @@ "dataset": "instructions.json", "label": "APPROX", "notes": "files holding them" + }, + { + "key": "LANGUAGE_FEATURE_ROWS", + "cmd": "python3 docs-out/tools/gen_T210.py --count rows", + "dataset": "02.7-language-features.md", + "label": "APPROX", + "notes": "T210" + }, + { + "key": "BINDER_CLASSES", + "cmd": "python3 docs-out/tools/gen_T26.py --count binders", + "dataset": "binders.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "BINDER_CLASSES_PS", + "cmd": "python3 docs-out/tools/gen_T26.py --count ps_binders", + "dataset": "binders.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "BINDER_CREATION_SITES", + "cmd": "python3 docs-out/tools/gen_T26.py --count created", + "dataset": "binders.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "RUNTIME_OP_TYPES", + "cmd": "python3 docs-out/tools/gen_T26.py --count optypes", + "dataset": "types.json", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "OPERATOR_KINDS", + "cmd": "python3 docs-out/tools/gen_T28.py --count operators", + "dataset": "tables.json", + "label": "APPROX", + "notes": "T28" + }, + { + "key": "CONVERSION_RANKS", + "cmd": "python3 docs-out/tools/gen_T28.py --count ranks", + "dataset": "enums.json", + "label": "APPROX", + "notes": "T28" + }, + { + "key": "SPECIAL_VARIABLE_CONSTANTS", + "cmd": "python3 docs-out/tools/gen_T29.py --count constants", + "dataset": "auto-variables.json", + "label": "APPROX", + "notes": "T29" + }, + { + "key": "EXECCONTEXT_MEMBERS", + "cmd": "python3 docs-out/tools/gen_T29.py --count execcontext", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "T29" } ] \ No newline at end of file diff --git a/docs-out/data/numbers.json b/docs-out/data/numbers.json index 229337d..8bb5f48 100644 --- a/docs-out/data/numbers.json +++ b/docs-out/data/numbers.json @@ -520,5 +520,86 @@ "evidence": "E1", "label": "APPROX", "notes": "files holding them" + }, + { + "key": "LANGUAGE_FEATURE_ROWS", + "value": 39, + "dataset": "02.7-language-features.md", + "command": "python3 docs-out/tools/gen_T210.py --count rows", + "evidence": "E1", + "label": "APPROX", + "notes": "T210" + }, + { + "key": "BINDER_CLASSES", + "value": 30, + "dataset": "binders.json", + "command": "python3 docs-out/tools/gen_T26.py --count binders", + "evidence": "E1", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "BINDER_CLASSES_PS", + "value": 26, + "dataset": "binders.json", + "command": "python3 docs-out/tools/gen_T26.py --count ps_binders", + "evidence": "E1", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "BINDER_CREATION_SITES", + "value": 164, + "dataset": "binders.json", + "command": "python3 docs-out/tools/gen_T26.py --count created", + "evidence": "E1", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "RUNTIME_OP_TYPES", + "value": 51, + "dataset": "types.json", + "command": "python3 docs-out/tools/gen_T26.py --count optypes", + "evidence": "E1", + "label": "APPROX", + "notes": "T26" + }, + { + "key": "OPERATOR_KINDS", + "value": 75, + "dataset": "tables.json", + "command": "python3 docs-out/tools/gen_T28.py --count operators", + "evidence": "E1", + "label": "APPROX", + "notes": "T28" + }, + { + "key": "CONVERSION_RANKS", + "value": 38, + "dataset": "enums.json", + "command": "python3 docs-out/tools/gen_T28.py --count ranks", + "evidence": "E1", + "label": "APPROX", + "notes": "T28" + }, + { + "key": "SPECIAL_VARIABLE_CONSTANTS", + "value": 70, + "dataset": "auto-variables.json", + "command": "python3 docs-out/tools/gen_T29.py --count constants", + "evidence": "E1", + "label": "APPROX", + "notes": "T29" + }, + { + "key": "EXECCONTEXT_MEMBERS", + "value": 124, + "dataset": "members-tierA.json", + "command": "python3 docs-out/tools/gen_T29.py --count execcontext", + "evidence": "E1", + "label": "APPROX", + "notes": "T29" } ] \ No newline at end of file diff --git a/docs-out/data/subject-keys.json b/docs-out/data/subject-keys.json index 568007b..24412dc 100644 --- a/docs-out/data/subject-keys.json +++ b/docs-out/data/subject-keys.json @@ -1,4 +1,7 @@ { + "arith.int_divide": "", + "arith.int_overflow": "", + "arith.zero_divide": "", "ast.assignment.rhs_normalization": "", "ast.constant.evaluation": "", "ast.find.nested_skip": "", @@ -12,6 +15,8 @@ "ast.ternary.visitor_gate": "", "ast.visit.pre_order_post_hook": "", "ast.visit.v2_nodes": "", + "binder.count": "", + "closure.captures": "", "compiler.foreach.enumeration": "", "compiler.log_segment": "", "compiler.loop.nonlocal_break": "", @@ -21,7 +26,20 @@ "compiler.statement_dispatch": "", "compiler.switch.clauses": "", "compiler.visit_method_count": "", + "convert.constrained_gate": "", + "convert.double_format": "", + "convert.string_to_number": "", + "destructuring.shape": "", + "enumeration.dictionary": "", + "enumeration.string": "", + "erroraction.default": "", + "erroraction.throw_statement": "", + "hashtable.add.comparer": "", + "hashtable.literal.duplicate": "", "interpreter.max_parameters": "", + "loopflow.empty_label": "", + "operator.precedence.levels": "", + "output.unroll_depth": "", "parser.array_type.max_dimensions": "", "parser.assignment.rhs": "", "parser.binary_operators.associativity": "", @@ -39,7 +57,18 @@ "parser.statement.terminator_after_block": "", "parser.switch.flags_matching": "", "parser.using_assembly.resolution_time": "", + "pipelineresult.bool": "", + "repeat.restricted_cap": "", "restricted.default_variables": "", + "scope.allscope.copy": "", + "scope.automatic.slots": "", + "scope.global.defaults": "", + "scope.preference.slots": "", + "scope.read.private_skip": "", + "scope.read.unqualified": "", + "scope.strict.version1_exception": "", + "scope.strict.walk": "", + "scope.write.unqualified": "", "semantic.array_type.max_depth": "", "semantic.order": "", "semantic.unresolved_type_error": "", @@ -71,9 +100,13 @@ "tokenizer.requires.recognition": "", "tokenizer.string.single_quote_escape": "", "tokenizer.subexpr.prescan": "", + "trap.continue": "", + "truthiness.list": "", + "truthiness.string": "", "typedefiner.run_time": "", "typeresolver.order": "", "variableanalysis.force_dynamic": "", "variableanalysis.pessimizing_cmdlets": "", - "variableanalysis.unanalyzed": "" + "variableanalysis.unanalyzed": "", + "variablepath.drive_rule": "" } \ No newline at end of file diff --git a/docs-out/tools/annotations/T26-binders.json b/docs-out/tools/annotations/T26-binders.json new file mode 100644 index 0000000..fd713cc --- /dev/null +++ b/docs-out/tools/annotations/T26-binders.json @@ -0,0 +1,135 @@ +{ + "_note": "Hand-written annotations for BINDER-CATALOG.md (task T26). 'read' = body of Bind/Fallback was read (FACT-SRC); 'names' = only member names and compiler usage read (UNVERIFIED detail).", + "binders": { + "PSEnumerableBinder": [ + "read", + "Convert a value to an IEnumerator following the enumeration rule (CF-PIP-001): AutomationNull = empty; null, string, PSObject-without-base, IDictionary, XmlNode = not enumerable (null result); arrays, DataTable rows, COM enumerables, IEnumerable (typed enumerator preferred), IEnumerable, IEnumerator = enumerated.", + "foreach, switch, pipelines, @(), operators that accept collections" + ], + "PSToObjectArrayBinder": [ + "read", + "Value to object[]: AutomationNull = empty array; non-enumerable = one-element array; List = ToArray; any enumerable = array of its elements.", + "`@( \u2026 )` over a single value" + ], + "PSPipeWriterBinder": [ + "read", + "Write a statement value to the output pipe: AutomationNull writes nothing; non-enumerable written as one object (mutable value types copied via PSVariableAssignmentBinder); enumerable unrolled one level and disposed.", + "every bare expression statement (CallAddPipe)" + ], + "PSArrayAssignmentRHSBinder": [ + "read", + "Shape the right side of `$a,$b,$c = \u2026` to exactly n elements: list of n = as is; fewer = padded with $null; more = last target receives the rest as a slice; non-list = first target gets it, others $null.", + "array destructuring assignment" + ], + "PSToStringBinder": [ + "names", + "Convert a value to string for expandable strings, -f arguments and file names (ToString via the engine string parser; InvokeToString helper).", + "expandable strings, format operator, redirection file names" + ], + "PSPipelineResultToBoolBinder": [ + "read", + "Captured pipeline list to bool: 0 elements = false; 1 element = that element converted to bool; more than 1 = true.", + "conditions of if/while/for/switch/do" + ], + "PSInvokeDynamicMemberBinder": [ + "names", + "Invoke a method whose name is computed at run time (`$o.$name(args)`).", + "dynamic member invocation" + ], + "PSGetDynamicMemberBinder": [ + "names", + "Get a member whose name is computed at run time (`$o.$name`).", + "dynamic member read" + ], + "PSSetDynamicMemberBinder": [ + "names", + "Set a member whose name is computed at run time.", + "dynamic member write" + ], + "PSSwitchClauseEvalBinder": [ + "names", + "Evaluate a non-constant switch clause expression against `$_` according to the switch flags (regex, wildcard, exact, case-sensitive, script block).", + "switch clauses that are not constants" + ], + "PSAttributeGenerator": [ + "names", + "Create an attribute instance from positional and named argument values (CreateInstanceBinder); compiled lambda cached per call shape.", + "attribute creation for parameters/functions" + ], + "PSCustomObjectConverter": [ + "names", + "Convert an ordered dictionary to a PSCustomObject.", + "`[pscustomobject]@{\u2026}`" + ], + "PSDynamicConvertBinder": [ + "names", + "Conversion whose target type is resolved at run time (type name unresolved at compile time).", + "`[SomeType]$x` with late-bound type" + ], + "PSVariableAssignmentBinder": [ + "names", + "Copy mutable value types when a value is stored in a variable or written to the pipe (so later mutation does not alias); also notes types with instance members.", + "assignment to object-typed variables, pipe writes" + ], + "PSBinaryOperationBinder": [ + "read", + "Binary operators on arbitrary operands: dispatch on the operation, then rules per operand types (numeric promotion, string/array/hashtable arithmetic, comparison with enumerable filtering, user-defined op_* methods). See OPERATOR-TABLE.md.", + "every non-native binary operator" + ], + "PSUnaryOperationBinder": [ + "names", + "Unary Not, OnesComplement (-bnot), plus, minus, increment, decrement (members Not, BNot, UnaryPlus, UnaryMinus, IncrDecr).", + "unary operators" + ], + "PSConvertBinder": [ + "read", + "Cast/conversion: AutomationNull converts to the default value of the target; otherwise FigureConversion(value, type) and invoke the converter with invariant culture; binding is restricted by a version number and the language mode; ThrowNoConversion raises the cast error.", + "`[type]value`, typed variables, parameter conversions" + ], + "PSGetIndexBinder": [ + "names", + "Index read: dictionaries, arrays (negative indices count from the end, multi-dimensional arrays, slicing by index lists), strings, objects with an indexer, null-conditional; errors via CannotIndexTarget.", + "`$x[i]`, `?[`" + ], + "PSSetIndexBinder": [ + "names", + "Index write with the same shapes as read, plus value conversion.", + "`$x[i] = v`" + ], + "PSGetMemberBinder": [ + "names", + "Property/field read: instance (ETS) members, adapted .NET members, hashtable/dictionary keys, member enumeration over collections, reserved intrinsic members; strict-mode property-missing errors; language-mode checks.", + "`$x.Name`, `[T]::Name`" + ], + "PSGetMemberBinder.ReservedMemberBinder": [ + "names", + "Handles intrinsic reserved members of PSObject-style access (nested binder).", + "reserved member names" + ], + "PSSetMemberBinder": [ + "names", + "Property/field write with value conversion to the member type, instance members and adapted members.", + "`$x.Name = v`" + ], + "PSInvokeMemberBinder": [ + "names", + "Method call: overload resolution (FindBestMethod with conversion ranks), .Where()/.ForEach() intrinsic methods on collections, method call on each element of a collection, parameterized properties, COM, member-invocation logging.", + "`$x.Method(args)`, `[T]::Method(args)`" + ], + "PSInvokeBinder": [ + "names", + "Invoke a callable value.", + "invocation of delegates/script blocks" + ], + "PSCreateInstanceBinder": [ + "names", + "Constructor call for `[T]::new(args)` / `New-Object`-style creation; rejects by-ref-like and (optionally) non-public types.", + "`[T]::new(\u2026)`" + ], + "PSInvokeBaseCtorBinder": [ + "names", + "Call a base-class constructor from a class constructor.", + "`: base(args)` in class constructors" + ] + } +} \ No newline at end of file diff --git a/docs-out/tools/annotations/T28-operators.json b/docs-out/tools/annotations/T28-operators.json new file mode 100644 index 0000000..80e5e3c --- /dev/null +++ b/docs-out/tools/annotations/T28-operators.json @@ -0,0 +1,48 @@ +{ +"_note": "Hand-written annotations for OPERATOR-TABLE.md (task T28), FACT-SRC from Compiler.cs (VisitBinaryExpression/VisitUnaryExpression), Binders.cs (PSBinaryOperationBinder, PSUnaryOperationBinder) and parserutils.cs (ParserOps). Keys are TokenKind names; 'groups' give shared text.", +"groups": { +"add": ["Dynamic site PSBinaryOperationBinder(Add) (double,double: native add)", "null + x gives x; numeric + numeric uses the wider type with overflow to double; numeric + string converts the string to a number (Parser.ScanNumber); string + any (and char + any) concatenates using the right operand's string form; array/enumerable + x appends (array concatenation); hashtable + hashtable merges, hashtable + other is an error; otherwise a user-defined op_Addition"], +"sub": ["Dynamic site PSBinaryOperationBinder(Subtract)", "numeric (wider type, overflow to double); string left operand converted to number; otherwise op_Subtraction"], +"mul": ["Dynamic site PSBinaryOperationBinder(Multiply)", "null * x is null; numeric; numeric * string converts the string; string * n repeats the string; array/enumerable * n repeats the elements; otherwise op_Multiply"], +"div": ["Dynamic site PSBinaryOperationBinder(Divide)", "null / x treats null as 0; int/int gives an int when the division is exact, otherwise a double; division by zero raises a RuntimeException; string operands converted to numbers; otherwise op_Division"], +"rem": ["Dynamic site PSBinaryOperationBinder(Modulo)", "null % x treats null as 0; integer remainder; remainder by zero raises a RuntimeException; otherwise op_Modulus"], +"shift": ["Dynamic site PSBinaryOperationBinder(LeftShift/RightShift)", "integer shift; operand type int, uint, long or ulong chosen from the wider operand"], +"bitwise": ["Dynamic site PSBinaryOperationBinder(And/Or/ExclusiveOr)", "integer bitwise operation on int, uint, long or ulong chosen from the wider operand"], +"logical": ["AndAlso/OrElse over bool conversions (-and, -or); NotEqual over two bools (-xor)", "operands converted with PowerShell truthiness (IsTrue); -and/-or short-circuit; -xor evaluates both"], +"cmp_eq": ["Dynamic site PSBinaryOperationBinder(Equal/NotEqual) with ignore-case flag (i-variants and the plain operator ignore case, c-variants do not)", "null handling; enumerable left operand filters (returns matching elements) unless scalar compare; string left operand: ordinal-culture string compare with right operand converted to string; char left operand ignoring case; numeric compare; otherwise Equals, then Equals after converting the right operand to the left operand's type (an InvalidCast gives false)"], +"cmp_ord": ["Dynamic site PSBinaryOperationBinder(GreaterThan/LessThan/...) with ignore-case flag", "same dispatch as equality; non-numeric left operands use LanguagePrimitives.Compare (right operand converted to the left type; left operand must be comparable)"], +"like": ["ParserOps.LikeOperator(context, position, left, right, operator)", "wildcard match; array left operand filters; constant right operand pre-built to a WildcardPattern"], +"match": ["ParserOps.MatchOperator(context, position, left, right, notMatch, ignoreCase)", "regex match; scalar left operand sets $Matches (hashtable, case-insensitive keys; numeric group names become int keys); array left operand filters elements"], +"replace": ["ParserOps.ReplaceOperator(context, position, left, right, ignoreCase)", "right operand is an array of 1..2 elements (pattern, replacement); replacement string or script block (MatchEvaluator); more than 2 elements is an error"], +"split": ["ParserOps.SplitOperator / UnarySplitOperator(context, position, left, right, ignoreCase)", "regex split by default (SimpleMatch option for literal); optional limit and options; predicate script block form; unary form splits on whitespace"], +"join": ["ParserOps.JoinOperator / UnaryJoinOperator(context, position, left, right)", "joins elements converted to strings with the separator; unary form joins with an empty separator"], +"contains": ["GenerateCallContains: ParserOps.ContainsOperatorCompiled with enumerator site and Equal binary site (scalar compare)", "collection membership by PowerShell equality; -in/-notin swap the operands; -notcontains/-notin negate"], +"is": ["TypeOps.IsInstance / TypeIs on the base object; -isnot negates", "right operand is a type; PSObject/PSCustomObject special cases"], +"as": ["TypeOps.AsOperator", "converts or returns $null"], +"format": ["StringOps.FormatOperator(format string, argument)", "string.Format with the right operand as arguments (array elements spread)"], +"range": ["ParserOps.RangeOperator / range enumerator", "integer ranges up and down; single-character operands give a character range"], +"coalesce": ["Compiler.Coalesce helper", "left operand unless it is $null"], +"not": ["Dynamic site PSUnaryOperationBinder(Not)", "logical negation of truthiness"], +"bnot": ["Dynamic site PSUnaryOperationBinder(OnesComplement)", "integer bitwise complement"], +"unary_arith": ["0 + x or 0 - x through PSBinaryOperationBinder", "unary plus and minus are implemented as binary add/subtract from 0"], +"incdec": ["Compiler.CompileIncrementOrDecrement via IAssignableValue", "prefix returns the new value; postfix returns the old value (null postfix result is 0); needs an assignable operand"], +"assign": ["CompileAssignment / ReduceAssignment", "= stores; op= applies the dynamic binary operation to the current value then stores; ??= stores only when the current value is $null"], +"special": ["Parser/tokenizer level", "member access, static member access, invocation/call/dot-source; no expression-tree operator"], +"redirect": ["Redirection objects (FileRedirection, MergingRedirection)", "stream redirection, not an expression operator"] +}, +"kinds": { +"Plus":"add","Minus":"sub","Multiply":"mul","Divide":"div","Rem":"rem","Shl":"shift","Shr":"shift","Band":"bitwise","Bor":"bitwise","Bxor":"bitwise", +"And":"logical","Or":"logical","Xor":"logical","Not":"not","Exclaim":"not","Bnot":"bnot", +"Ieq":"cmp_eq","Ine":"cmp_eq","Ceq":"cmp_eq","Cne":"cmp_eq", +"Ige":"cmp_ord","Igt":"cmp_ord","Ilt":"cmp_ord","Ile":"cmp_ord","Cge":"cmp_ord","Cgt":"cmp_ord","Clt":"cmp_ord","Cle":"cmp_ord", +"Ilike":"like","Inotlike":"like","Clike":"like","Cnotlike":"like", +"Imatch":"match","Inotmatch":"match","Cmatch":"match","Cnotmatch":"match", +"Ireplace":"replace","Creplace":"replace","Isplit":"split","Csplit":"split","Join":"join", +"Icontains":"contains","Inotcontains":"contains","Ccontains":"contains","Cnotcontains":"contains","Iin":"contains","Inotin":"contains","Cin":"contains","Cnotin":"contains", +"Is":"is","IsNot":"is","As":"as","Format":"format","DotDot":"range","QuestionQuestion":"coalesce", +"PlusPlus":"incdec","MinusMinus":"incdec","PostfixPlusPlus":"incdec","PostfixMinusMinus":"incdec", +"Equals":"assign","PlusEquals":"assign","MinusEquals":"assign","MultiplyEquals":"assign","DivideEquals":"assign","RemainderEquals":"assign","QuestionQuestionEquals":"assign", +"Dot":"special","ColonColon":"special","QuestionDot":"special","Ampersand":"special","Pipe":"special","Colon":"special","QuestionMark":"special","Comma":"special","QuestionLBracket":"special", +"Redirection":"redirect","RedirectInStd":"redirect" +} +} diff --git a/docs-out/tools/gen_T210.py b/docs-out/tools/gen_T210.py new file mode 100644 index 0000000..476cfe4 --- /dev/null +++ b/docs-out/tools/gen_T210.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""gen_T210.py - counts for 02-language/02.7-language-features.md (task T210). + +Purpose : the feature table of 02.7 is hand-maintained; this tool counts its rows so the number in the text is registered. +Inputs : docs-out/02-language/02.7-language-features.md +Run : python3 docs-out/tools/gen_T210.py --count rows +Language: Python 3 (D-002) +""" +import os +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +p = os.path.join(ROOT, "docs-out", "02-language", "02.7-language-features.md") +t = open(p, encoding="utf-8").read().split("## 5. Data model")[1].split("## 6.")[0] +rows = [l for l in t.split("\n") if l.startswith("| ") and not l.startswith("| Feature") and not l.startswith("|---")] +print(len(rows)) diff --git a/docs-out/tools/gen_T26.py b/docs-out/tools/gen_T26.py new file mode 100644 index 0000000..7480d21 --- /dev/null +++ b/docs-out/tools/gen_T26.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""gen_T26.py - generated catalogs of task T26 (rule 9). + +Purpose : (1) 02-language/BINDER-CATALOG.md - every binder class (data/binders.json): base class, source range, + creation sites, hand annotation (tools/annotations/T26-binders.json); + (2) region of 02-language/RUNTIME-OPERATIONS.md - every runtime helper type of + engine/runtime/Operations/*.cs and engine/lang/parserutils.cs with range and member count. +Inputs : data/binders.json, data/types.json, data/members-tierA.json, annotation file +Outputs : docs-out/02-language/BINDER-CATALOG.md, region of RUNTIME-OPERATIONS.md +Run : python3 docs-out/tools/gen_T26.py [--count binders|optypes|created] +Label : APPROX (scanner data); annotations FACT-SRC where marked 'read', UNVERIFIED detail where 'names'. +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def esc(s): + return s.replace("|", "\\|") + + +def main(): + b = J("binders.json") + ann = json.load(open(os.path.join(OUT, "tools", "annotations", "T26-binders.json")))["binders"] + types = J("types.json") + mem = {(m["file"], m["full_name"]): m["members"] for m in J("members-tierA.json")} + ops = [t for t in types if "/runtime/Operations/" in t["file"] or t["file"].endswith("lang/parserutils.cs")] + created = sum(len(x.get("created_or_obtained_at", [])) for x in b) + if len(sys.argv) > 2 and sys.argv[1] == "--count": + print({"binders": len(b), "optypes": len(ops), "created": created, "ps_binders": sum(1 for x in b if "Language." in x["name"])}[sys.argv[2]]) + return 0 + L = [] + w = L.append + w("""--- +id: LANG-BINDERCAT-001 +title: Binder catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/runtime/Binding/Binders.cs, src/System.Management.Automation/engine/ComInterop/ComBinder.cs, src/System.Management.Automation/engine/ComInterop/ComInvokeAction.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/runtime/_folder.md] +generated_from: [data/binders.json] +--- + +# Binder catalog + +> STATIC-ONLY — behavior not run-verified. The table is `GENERATED` by `docs-out/tools/gen_T26.py` from `data/binders.json` (`APPROX`) joined with hand annotations (`docs-out/tools/annotations/T26-binders.json`). Annotation level **read** = the binder's bind/fallback body was read (`FACT-SRC`); **names** = only member names and the compiler's use were read, details `UNVERIFIED` (VB-020). Narrative: [02.6-types-and-operators.md](02.6-types-and-operators.md), [RUNTIME-OPERATIONS.md](RUNTIME-OPERATIONS.md), [IR-SPEC.md](IR-SPEC.md) §5. +""") + w("## 1. Purpose and scope [SPEC]\n") + w(f"""A *binder* is a DLR object that turns an operation on runtime values into code, and caches the result per call site by operand types. Every non-trivial operation in compiled PowerShell code (operators, casts, member access, indexing, enumeration, output writes) is a dynamic call site with one of these binders. The scanner found {len(b)} binder classes ({sum(1 for x in b if 'Language.' in x['name'])} in `Binders.cs`, the rest COM interop), with {created} creation or lookup sites. +""") + w("## 2. Source map\n") + w("| path | role |\n|---|---|\n| `src/System.Management.Automation/engine/runtime/Binding/Binders.cs` | all PowerShell binders (8,015 lines) |\n| `src/System.Management.Automation/engine/ComInterop/ComBinder.cs`, `ComInvokeAction.cs` | COM member and invoke binders (out of scope, listed for completeness) |\n") + w("## 3. Binder table\n") + w("| Binder | Base | Operation (annotation) | Level | Used for | Creation sites | Source |") + w("|---|---|---|---|---|---:|---|") + miss = [] + for x in sorted(b, key=lambda z: (z["file"], z["start"])): + short = x["name"].split("Language.")[-1].split("ComInterop.")[-1] + a = ann.get(short) + if a is None: + if "ComInterop" in x["name"]: + a = ["names", "COM interop binder (out of scope).", "COM objects"] + else: + miss.append(short) + a = ["?", "?", "?"] + base = ", ".join(f"`{q}`" for q in x["bases"]) or "—" + w(f"| `{short}` | {base} | {esc(a[1])} | {a[0]} | {esc(a[2])} | {len(x.get('created_or_obtained_at', []))} | `{x['file']}::{short.split('.')[-1]}@L{x['start']}-L{x['end']}` |") + w(f"\nRows: **{len(b)}** (`GENERATED`). Binders without annotation: {', '.join(miss) or 'none'}.\n") + w("## 4. Common mechanics [IMPL]\n") + w("""- **Defer when values are missing**: a binder first checks `target.HasValue`; if an operand is a `PSObject` wrapper with a different base object it defers to a PSObject-aware path (`DeferForPSObject`), except `Add` on enumerables. +- **Restrictions**: each produced rule is guarded by type restrictions (`PSGetTypeRestriction`) and, for conversions, by a binder *version* and the language mode, so a language-mode change or a type-table change invalidates cached rules (`InvalidateCache`, version counters; `ExecutionContext.LanguageMode` setter). +- **Per-site caches**: each emitted dynamic expression has its own `CallSite`; both execution tiers use them ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md) EQ-02). +- **Fallback with error suggestion**: binders pass an `errorSuggestion` so that "operation not supported" throws a PowerShell error instead of a DLR error. +""") + w("## 5. Guidance for MicroteamShell [LESSON]\n") + w("""- **LSN-BIND-01** [ADAPT] (area: NLD-RUN) Replace DLR binders by explicit dispatch functions per operation (`Add(left, right)`, `Convert(value, type)`, `GetMember(target, name)`, `Index(target, args)`) with a small per-site cache only if profiling shows a need. Why: the DLR is a mechanism, the observable behavior is the rule set. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`. +- **LSN-BIND-02** [REPLICATE] (area: NLD-TYP) Keep the dedicated helper operations (enumerate, to-object-array, write-to-pipe, array-assignment shaping, pipeline-result-to-bool); they encode language rules. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSEnumerableBinder@L518-L780`, `PSPipeWriterBinder@L841-L926`, `PSArrayAssignmentRHSBinder@L935-L1042`, `PSPipelineResultToBoolBinder@L1119-L1178`. +- **LSN-BIND-03** [AVOID] (area: NLD-RUN) Do not port COM binders and `PSGetMemberBinder`'s COM/ETS-adapter paths in the first versions. Replace with a plain reflection member lookup. Evidence: `src/System.Management.Automation/engine/ComInterop/ComBinder.cs::ComBinder@L1-L378`. DEV: DEV-BIND-050 +""") + w("Verification: binder list reconciles with `data/binders.json` (counts above); `PSEnumerableBinder`, `PSToObjectArrayBinder`, `PSPipeWriterBinder`, `PSArrayAssignmentRHSBinder`, `PSPipelineResultToBoolBinder`, `PSConvertBinder.FallbackConvert`, `PSBinaryOperationBinder` (dispatch, add, multiply, divide/remainder, equality, numeric) bodies were read; the others by member list and compiler use.\n") + open(os.path.join(OUT, "02-language", "BINDER-CATALOG.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + # runtime ops region + p = os.path.join(OUT, "02-language", "RUNTIME-OPERATIONS.md") + if os.path.exists(p): + rows = ["| Type | File | Lines | Members (scan) |", "|---|---|---|---:|"] + for t in sorted(ops, key=lambda z: (z["file"], z["start"])): + n = len(mem.get((t["file"], t["full_name"]), [])) + rows.append(f"| `{t['full_name'].split('Automation.')[-1]}` | `{t['file'].split('/')[-1]}` | L{t['start']}-L{t['end']} | {n} |") + txt = open(p, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()", re.S) + if pat.search(txt): + open(p, "w", encoding="utf-8").write(pat.sub(lambda m: m.group(1) + "\n" + "\n".join(rows) + "\n" + m.group(2), txt)) + print("BINDER-CATALOG rows", len(b), "missing annotation", miss, "optypes", len(ops)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T28.py b/docs-out/tools/gen_T28.py new file mode 100644 index 0000000..4beeb80 --- /dev/null +++ b/docs-out/tools/gen_T28.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""gen_T28.py - generated documents of task T28 (rule 9). + +Purpose : (1) 02-language/OPERATOR-TABLE.md - every operator token kind (binary, unary, assignment, special) + with text, precedence level, flags and implementation route (hand annotations); + (2) region of 02-language/CONVERSION-RULES.md - ConversionRank table from data/enums.json. +Inputs : data/enums.json, data/tables.json (via gen_T21 parsers), tools/annotations/T28-operators.json +Outputs : docs-out/02-language/OPERATOR-TABLE.md, region in CONVERSION-RULES.md +Run : python3 docs-out/tools/gen_T28.py [--count operators|ranks] +Label : APPROX (token tables parsed from verbatim source text; precedence from TokenFlags). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +sys.path.insert(0, os.path.join(OUT, "tools")) +import gen_T21 as g # noqa: E402 + +TKN = g.TKN +LP = "src/System.Management.Automation/engine/LanguagePrimitives.cs" + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def operators(): + tb = J("tables.json") + en = {e["full_name"].split(".")[-1]: e for e in J("enums.json") if e["file"] == TKN} + kinds = en["TokenKind"]["members"] + sfl = [re.sub(r"\s+", " ", x) for x in g.split_top(g.strip_comments(tb[f"{TKN}::s_staticTokenFlags"]["text"]))] + txt = [g.unq(x) for x in g.split_top(g.strip_comments(tb[f"{TKN}::s_tokenText"]["text"]))] + flags = {m["name"]: int(m["value"], 16) for m in en["TokenFlags"]["members"] if m["name"].startswith("BinaryPrecedence") and m["name"] != "BinaryPrecedenceMask"} + rows = [] + for m in kinds: + v = int(m["value"]) + f = sfl[v] + cls = [c for c in ("BinaryOperator", "UnaryOperator", "AssignmentOperator", "SpecialOperator", "PrefixOrPostfixOperator") if c in f] + if not cls and m["name"] not in ("Comma", "Redirection", "RedirectInStd", "Pipe", "Ampersand"): + continue + lvl = [k for k in flags if "TokenFlags." + k in f] + rows.append({"name": m["name"], "text": txt[v], "classes": cls, "level": flags[lvl[0]] if lvl else None, + "levelname": lvl[0].replace("BinaryPrecedence", "") if lvl else "", "fold": "CanConstantFold" in f, + "restricted": "DisallowedInRestrictedMode" in f, "line": m["line"]}) + return rows + + +def main(): + ann = json.load(open(os.path.join(OUT, "tools", "annotations", "T28-operators.json"))) + rows = operators() + en = {e["full_name"].split(".")[-1]: e for e in J("enums.json") if e["file"] == LP} + ranks = en["ConversionRank"]["members"] + if len(sys.argv) > 2 and sys.argv[1] == "--count": + print(len(rows) if sys.argv[2] == "operators" else len(ranks)) + return 0 + unmapped = [r["name"] for r in rows if r["name"] not in ann["kinds"]] + L = [] + w = L.append + w("""--- +id: LANG-OPTABLE-001 +title: Operator table +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-OP-001] +sources: [src/System.Management.Automation/engine/parser/token.cs, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/runtime/Binding/Binders.cs, src/System.Management.Automation/engine/lang/parserutils.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/enums.json, data/tables.json] +--- + +# Operator table + +> STATIC-ONLY — behavior not run-verified. Rows are `GENERATED` by `docs-out/tools/gen_T28.py` from the token tables (`APPROX`) joined with hand annotations (`docs-out/tools/annotations/T28-operators.json`, `FACT-SRC`). Narrative: [02.6-types-and-operators.md](02.6-types-and-operators.md); conversions used by operators: [CONVERSION-RULES.md](CONVERSION-RULES.md); parsing: [02.2-parser.md](02.2-parser.md) §7.2. +""") + w("## 1. Purpose and scope [SPEC]\n") + w("One table for every operator token kind: spelling, role, precedence level, associativity, whether the compiler may fold it at compile time, and how it is evaluated. The precedence rule is canonical fact CF-OP-001 (stated in §3).\n") + w("## 2. Source map\n") + w("| path | role |\n|---|---|") + w("| `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752` | precedence and operator-class flags |") + w("| `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950` | precedence climbing |") + w("| `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422` | binary operation semantics |") + w("| `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSUnaryOperationBinder@L3427-L3738` | unary operation semantics |") + w("| `src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1760` | `-like`, `-match`, `-replace`, `-split`, `-join`, range helpers |\n") + w("## 3. Precedence and associativity (CF-OP-001)\n") + w("**Operator precedence and associativity.** Binary operators are parsed by precedence climbing and **all are left-associative**; from lowest to highest precedence the levels are: logical (`-and`, `-or`, `-xor`) = 1; bitwise (`-band`, `-bor`, `-bxor`) = 2; comparison group (all `-eq`/`-ne`/`-gt`/`-ge`/`-lt`/`-le`, `-like`, `-match`, `-replace`, `-contains`, `-in` families with their `i`/`c` variants, `-is`, `-isnot`, `-as`, `-split`, `-join`, `-shl`, `-shr`) = 5; null-coalescing `??` = 7; additive (`+`, `-`) = 9; multiplicative (`*`, `/`, `%`) = 10; format `-f` = 12; range `..` = 13. The comma (array-literal) operator binds tighter than every binary operator; the unary operators, casts and `++`/`--` bind tighter than the comma; the ternary `?:` (right-nested) and assignment (`=`, `+=`, `-=`, `*=`, `/=`, `%=`, `??=`, parsed at statement level with a statement on the right) bind looser than all binary operators; `&&` and `||` chain pipelines and are not expression operators. Operators spelled with a leading dash are recognized as operators only when the tokenizer is in expression mode.\n") + w("Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950` (reduce while new precedence is less than or equal), `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752` (level values), `Parser.ArrayLiteralRule@L6975-L7021`, `Parser.UnaryExpressionRule@L7029-L7226`, `Parser.ExpressionRule@L6696-L6790`, `Parser.PipelineChainRule@L5773-L5988`.\n") + w("## 4. Operator table\n") + w("Level = `BinaryPrecedence*` value (higher binds tighter); `fold` = the operator may be constant-folded; `restricted` = flagged `DisallowedInRestrictedMode`. Case: `i`-prefixed and plain forms ignore case, `c`-prefixed forms are case-sensitive (token texts of `i` forms are listed without the `i` for most operators, see [TOKEN-CATALOG.md](TOKEN-CATALOG.md) §3).\n") + w("| TokenKind | Text | Class | Level | Associativity | fold | restricted | Evaluation | Behavior (FACT-SRC) |") + w("|---|---|---|---:|---|---|---|---|---|") + for r in sorted(rows, key=lambda x: (x["level"] if x["level"] is not None else 99, x["name"])): + grp = ann["kinds"].get(r["name"]) + ev, beh = ann["groups"][grp] if grp else ("—", "—") + cls = "/".join(c.replace("Operator", "") for c in r["classes"]) or "other" + assoc = "left" if "BinaryOperator" in r["classes"] else ("right (nested)" if r["name"] == "QuestionMark" else "—") + w(f"| `{r['name']}` | `{r['text']}` | {cls} | {r['level'] if r['level'] is not None else '—'} | {assoc} | {'yes' if r['fold'] else ''} | {'yes' if r['restricted'] else ''} | {ev} | {beh} |") + w(f"\nRows: **{len(rows)}** (`GENERATED`). Kinds without annotation: {', '.join(unmapped) or 'none'}.\n") + w("## 5. Notes\n") + w("""- **Left-operand driven.** For binary operators the left operand's runtime type selects the rule; the right operand is converted to it where needed. `1 + "2"` is 3, `"1" + 2` is `"12"` (string concatenation). +- **`-eq`/`-ne` with arrays.** A collection on the left filters (returns matching elements); a `$null` on the left of `-eq` returns `$true` only for `$null`. +- **Constant folding** only for operators with the `CanConstantFold` flag; operators depending on execution context (`-split`, `-join`, `-like`) or culture (`-f`) are not folded (comment on `TokenFlags.CanConstantFold`). +- **`-and`/`-or`** short-circuit; **`-xor`** does not. **`-band`/`-bor`/`-bxor`/`-shl`/`-shr`** operate on integers of the wider operand type. +""") + w("## 6. Guidance for MicroteamShell [LESSON]\n") + w("""- **LSN-OP-01** [REPLICATE] (area: NLD-OP) Use the precedence levels and left associativity above; keep the comma tighter than binary operators and put ternary and assignment below. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950`. +- **LSN-OP-02** [REPLICATE] (area: NLD-OP) Choose the operation by the left operand's type and convert the right operand to it (CONVERSION-RULES); keep the `i`/`c` case variants and the array-filtering behavior of comparison operators. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder@L2119-L3422`. +- **LSN-OP-03** [ADAPT] (area: NLD-OP) Implement operators as ordinary functions with an explicit dispatch table instead of dynamic binding sites. Why: same observable results; no DLR. Evidence: `src/System.Management.Automation/engine/runtime/Binding/Binders.cs::PSBinaryOperationBinder.FallbackBinaryOperation@L2179-L2240`. +""") + w("Verification: operator rows generated from `s_staticTokenFlags`/`s_tokenText`; precedence levels cross-checked with the parser's reduce loop (`<=`) and with the generated table in [02.2-parser.md](02.2-parser.md) §7.2; evaluation text read from `PSBinaryOperationBinder` (`BinaryAdd`, `BinaryMultiply`, `BinarySubDivOrRem`, `CompareEQ`, `BinaryEqualityComparison`, `BinaryNumericOp`), `NumericOps.cs` (`IntOps`), `Compiler.VisitBinaryExpression` and `ParserOps` notes. `-shl`/`-shr`, bitwise and `-ge/-gt` internals and `PSUnaryOperationBinder` were read only at the dispatch level (`UNVERIFIED` below that).\n") + open(os.path.join(OUT, "02-language", "OPERATOR-TABLE.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + # ranks region + p = os.path.join(OUT, "02-language", "CONVERSION-RULES.md") + if os.path.exists(p): + out = ["| Rank name | Value | Source line |", "|---|---|---:|"] + for m in ranks: + out.append(f"| `{m['name']}` | `{m['value']}` | L{m['line']} |") + t = open(p, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()", re.S) + if pat.search(t): + open(p, "w", encoding="utf-8").write(pat.sub(lambda m: m.group(1) + "\n" + "\n".join(out) + "\n" + m.group(2), t)) + print("OPERATOR-TABLE rows", len(rows), "unmapped", unmapped, "ranks", len(ranks)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T29.py b/docs-out/tools/gen_T29.py new file mode 100644 index 0000000..55050f2 --- /dev/null +++ b/docs-out/tools/gen_T29.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""gen_T29.py - generated tables of 02-language/SCOPE-AND-VARIABLES.md (task T29, rule 9). + +Purpose : table of every name constant in SpecialVariables.cs (automatic, preference and engine variables) + with source line and reference count, from data/auto-variables.json; plus the two fixed-order enums. +Inputs : data/auto-variables.json, data/enums.json +Outputs : fills ... in docs-out/02-language/SCOPE-AND-VARIABLES.md +Run : python3 docs-out/tools/gen_T29.py [--count constants] +Label : APPROX (regex scan of `internal const string` fields; reference counts are textual). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +SV = "src/System.Management.Automation/engine/SpecialVariables.cs" + + +def main(): + a = json.load(open(os.path.join(DATA, "auto-variables.json")))["constants"] + if len(sys.argv) > 2 and sys.argv[1] == "--count": + if sys.argv[2] == "execcontext": + m = json.load(open(os.path.join(DATA, "members-tierA.json"))) + x = [t for t in m if t["file"].endswith("engine/ExecutionContext.cs") and t["full_name"].endswith(".ExecutionContext")][0] + print(len(x["members"])) + else: + print(len(a)) + return 0 + en = {e["full_name"].split(".")[-1]: e for e in json.load(open(os.path.join(DATA, "enums.json"))) if e["file"] == SV} + rows = ["| Field | Variable name | Line | References (APPROX) |", "|---|---|---:|---:|"] + for c in sorted(a, key=lambda x: x["line"]): + rows.append(f"| `{c['field']}` | `${c['name']}` | L{c['line']} | {c['reference_count']} |") + rows.append("") + for n in ("AutomaticVariable", "PreferenceVariable"): + e = en[n] + rows.append(f"`{n}` (`{SV}::{n}@L{e['start']}-L{e['end']}`): " + ", ".join(f"`{m['name']}`={m['value'] if m['value'] not in (None, '') else i}" for i, m in enumerate(e["members"]))) + rows.append("") + p = os.path.join(OUT, "02-language", "SCOPE-AND-VARIABLES.md") + t = open(p, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()", re.S) + if not pat.search(t): + print("region missing") + return 1 + open(p, "w", encoding="utf-8").write(pat.sub(lambda m: m.group(1) + "\n" + "\n".join(rows) + "\n" + m.group(2), t)) + print("special variable constants", len(a)) + + +if __name__ == "__main__": + sys.exit(main()) From 206fff13cbd255e95c595e8ae31df111740e31f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:37:59 +0000 Subject: [PATCH 2/2] Tidy PROGRESS.md task table Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01HnR7RpZuh9mvNMRTtzxTvk --- docs-out/PROGRESS.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs-out/PROGRESS.md b/docs-out/PROGRESS.md index 936bed1..f3df4ab 100644 --- a/docs-out/PROGRESS.md +++ b/docs-out/PROGRESS.md @@ -29,11 +29,10 @@ T21 tokenizer | T22 parser/grammar/errors | T23 AST | T24 semantic | T25 compile | T24 | `02-language/02.4-semantic-analysis.md` (check table generated by `tools/gen_T24.py`: 133 sites, 104 error IDs) | draft written; DEV-SEM-030, CON-009; checks pass | | T25 | `02-language/02.5-compiler-and-engine.md`, `COMPILATION-MAP.md` (`tools/gen_T25.py` + `docs-out/tools/annotations/T25-compilation-map.json`), `IR-SPEC.md` | draft written; DEV-ENG-040, CON-010, VB-015; checks pass | | T27 | `02-language/INSTRUCTION-SET.md` (`tools/gen_T27.py`), `TIERING-POLICY.md` (owns CF-CMP-001/002/003), `EQUIVALENCE-SPEC.md` | draft written; DEV-TIER-060, DEV-EQV-061, VB-013/014/016; checks pass | - -Next step: the coordinator documents of 02-language, then Phase 3 runtime (T31 runspace/pipeline, T32 command discovery, T33 parameter binding and cmdlet infrastructure, T34-T39) and the deeper reads listed in VB-017..VB-023. Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. - | T26 | `02-language/BINDER-CATALOG.md` (`tools/gen_T26.py` + `tools/annotations/T26-binders.json`), `RUNTIME-OPERATIONS.md` | draft; 30 binders listed (10 read in depth, 20 by names); DEV-BIND-050, DEV-RTOPS-051; CON-011/012; VB-020/021 | | T28 | `02-language/02.6-types-and-operators.md` (CF-PIP-001), `OPERATOR-TABLE.md` (CF-OP-001, `tools/gen_T28.py`), `CONVERSION-RULES.md` (CF-CNV-001) | draft; DEV-TYP-070; VB-018/019 | | T29 | `02-language/SCOPE-AND-VARIABLES.md` (CF-SCP-001, `tools/gen_T29.py`), `03-runtime/execution-context-and-session-state.md` | draft; DEV-SCP-080, DEV-CTX-081; VB-017 | | T210 | `02-language/02.7-language-features.md`, `HOWTO-extend-language.md`, `SCRIPT-BLOCKS.md` | draft; DEV-SB-090, DEV-FEAT-091; VB-022 | | T211 | `02-language/02.8-language-tooling.md` | draft; DEV-TOOLS-100; VB-023 | + +Next step: the coordinator documents of 02-language, then Phase 3 runtime (T31 runspace/pipeline, T32 command discovery, T33 parameter binding and cmdlet infrastructure, T34-T39) and the deeper reads listed in VB-017..VB-023. Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written.