diff --git a/docs-out/.work/cf/T21.json b/docs-out/.work/cf/T21.json new file mode 100644 index 0000000..2d481ad --- /dev/null +++ b/docs-out/.work/cf/T21.json @@ -0,0 +1,42 @@ +{ + "cfs": [ + { + "id": "CF-LEX-001", + "subject": "tokenizer mode switching rule (command vs expression mode, when it switches)", + "statement": "**Mode rule.** The tokenizer starts in Command mode; the parser switches to Expression mode when it parses an expression operand/operator position (including the first element of a pipeline when it is not a command), to Command mode when it parses a command's arguments or the body of `{ }`, `( )`, `$( )`, to TypeName mode inside `[...]` type literals, and to Signature mode for class/enum/method headers; every switch is restored in a `finally` block, and a switch discards and rescans a pending lookahead token unless its kind is `ParseModeInvariant`.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535", + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.CommandRule@L6444-L6684", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpressionRule@L6696-L6790" + ], + "owner": "02-language/02.1-tokenizer.md", + "status": "open" + }, + { + "id": "CF-LEX-002", + "subject": "numeric-literal rules (radix, suffixes, multipliers)", + "statement": "**Numeric literal rules.** A number is an optional sign (`-`, en/em dash/horizontal bar, or `+`; only when `AllowSignedNumbers`), then either `.digits[exponent]`, `0x` hex digits, `0b` binary digits, or decimal digits with optional `.digits` and `e/E[+|-]digits`; then an optional type suffix (`u s l d y n`, plus a second character only after `u`: `ul us uy`), then an optional multiplier `k m g t p` that **must be followed by `b`/`B`** (powers of 1024; the suffix comes before the multiplier); the literal must be followed by a token terminator (in Expression mode also by a binary-operator start character) or the text is not a number and is rescanned as a generic token. Value typing: no suffix \u2192 `int`, else `long`, else (decimal format only) `decimal`, else `double`; real literals \u2192 `double` unless suffix `d` (`decimal`); hex and binary are two's complement with the sign decided by the top digit (`0xFFFFFFFF` \u2192 `-1`, `0xFF` \u2192 `255`), a 16-hex-digit or 64-bit-digit literal without `u` becomes `long`; the multiplier is applied to the value before the range cast and failure to fit returns `false` (reported as `BadNumericConstant` in Expression mode, otherwise the text becomes a generic token).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163", + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882", + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumber@L3884-L3931" + ], + "owner": "02-language/02.1-tokenizer.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-LEX-001", + "baseline_behavior": "Tokenizer detects signature-block lookalikes and processes #requires comment lines at parse time.", + "proposed_behavior": "Omit both from the tokenizer; if wanted, handle pragmas in a dedicated post-tokenize pass.", + "benefit": "Keeps the tokenizer a pure text-to-token function; removes security-sensitive fuzzy matching from the hot path.", + "compatibility_cost": "Scripts using #requires get no version/module check; signed-script trailer detection unavailable.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T22.json b/docs-out/.work/cf/T22.json new file mode 100644 index 0000000..3037d53 --- /dev/null +++ b/docs-out/.work/cf/T22.json @@ -0,0 +1,45 @@ +{ + "cfs": [ + { + "id": "CF-GRM-001", + "subject": "statement, pipeline and command grammar; statement terminators", + "statement": "**Statement and terminator rule.** A statement list is parsed as: skip any run of newlines and semicolons; parse a statement; skip any run of newlines and semicolons; stop when the next token is `)` or `}` or no statement can be parsed. A terminator is **not** required between statements that end with a closing `}` (e.g. `if`, loops, `try`, `function`), because the list only skips optional terminators; a pipeline (or any statement parsed by `PipelineChainRule`) ends at newline, `;`, `)`, `}`, `&&`/`||` (chain), `&` (background) or end of input, and any other token after a complete pipeline element is `UnexpectedToken`. Newlines continue a construct only in these places: after `|`, before a `|` that starts the next line, after `&&` and `||`, after each binary or unary operator and after `,`, after `=` in an assignment, inside `( )`, after `?`/`:` of a ternary, between `}` and `elseif`/`else`/`catch`/`finally`, and after a backtick line continuation handled by the tokenizer.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineRule@L5990-L6160", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.CatchBlockRule@L5494-L5591" + ], + "owner": "02-language/GRAMMAR.ebnf.md", + "status": "open" + }, + { + "id": "CF-ERR-002", + "subject": "parse-error limits and recovery rule", + "statement": "**Parse-error limits and recovery rule.** The parser never throws for a syntax error: each error is appended to `Parser.ErrorList` after de-duplication (an error is dropped only if an earlier error has the same ID and the same start and end line and column), there is no limit on the number of errors, and parsing continues by returning partial or error nodes (`ErrorStatementAst`, `ErrorExpressionAst`), by pretending a missing closing token was present, or by skipping tokens to a synchronization token (`SyncOnError`, which tracks nesting of `()`, `{}` and `[]` and never consumes end of input); `ParseError.IncompleteInput` is true only when the error was reported with `ReportIncompleteInput` and the error position is at the end of the script apart from whitespace and comments. The only exceptions are an exhausted execution stack (the whole parse is retried once on a new thread, then `ScriptTooComplicated` is reported) and any other exception during `ParseFile`/`ParseInput`, which is wrapped as a `ParseException` with `UnrecoverableParserError`.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.SyncOnError@L366-L432", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8111-L8141" + ], + "owner": "02-language/ERROR-CATALOG.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-GRM-010", + "baseline_behavior": "The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing.", + "proposed_behavior": "Parsing is pure: directives are recorded in the AST and resolved in a later phase.", + "benefit": "Deterministic, side-effect-free parse; safe for tooling and untrusted text.", + "compatibility_cost": "Errors for a missing assembly appear at a later phase than in PowerShell.", + "affected_cfs": [ + "CF-GRM-001" + ], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T23.json b/docs-out/.work/cf/T23.json new file mode 100644 index 0000000..4630ace --- /dev/null +++ b/docs-out/.work/cf/T23.json @@ -0,0 +1,14 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-AST-020", + "baseline_behavior": "AST nodes carry mutable compiler caches (TupleIndex, RuntimeUsingIndex, TypeDefinitionAst.Type) and implement compile-time assignment protocols (ISupportsAssignment/IAssignableValue).", + "proposed_behavior": "AST stays purely syntactic; slot indexes and assignment code generation live in compiler side tables keyed by node.", + "benefit": "Immutable shareable trees; clean separation of parser and compiler; easier tooling reuse.", + "compatibility_cost": "None observable by scripts; internal refactor only.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T24.json b/docs-out/.work/cf/T24.json new file mode 100644 index 0000000..5246e3b --- /dev/null +++ b/docs-out/.work/cf/T24.json @@ -0,0 +1,14 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-SEM-030", + "baseline_behavior": "`using module` is resolved at parse time by running Get-Module -ListAvailable in a PowerShell instance.", + "proposed_behavior": "The directive is recorded; module import happens in a later phase.", + "benefit": "Parsing has no side effects or I/O.", + "compatibility_cost": "Missing-module errors appear later than in PowerShell.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T25.json b/docs-out/.work/cf/T25.json new file mode 100644 index 0000000..698d7b5 --- /dev/null +++ b/docs-out/.work/cf/T25.json @@ -0,0 +1,14 @@ +{ + "cfs": [], + "deviations": [ + { + "id": "DEV-ENG-040", + "baseline_behavior": "Compilation of a top-level script block runs an AMSI scan, admin-block check and suspicious-content hashing.", + "proposed_behavior": "Compilation is policy-free; an optional policy hook runs before execution.", + "benefit": "Smaller engine core; platform-independent.", + "compatibility_cost": "No AMSI integration or forced logging on suspicious content.", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/cf/T27.json b/docs-out/.work/cf/T27.json new file mode 100644 index 0000000..aafe823 --- /dev/null +++ b/docs-out/.work/cf/T27.json @@ -0,0 +1,68 @@ +{ + "cfs": [ + { + "id": "CF-CMP-001", + "subject": "compile-versus-interpret decision rule", + "statement": "**Compile-versus-interpret rule.** Every named block is turned into one expression-tree lambda. At creation of the delegate (`Compiler.CompileTree`), the tiering mode is `NeverCompile` if the script block's total statement count (`_stmtCount`) is greater than 300, otherwise `CompileOnDemand`; `AlwaysCompile` exists but no caller passes it. The delegate starts as an **interpreted** `LightLambda` with a call counter of 32 (`CompileOnDemand`, threshold -1 mapped to `DefaultCompilationThreshold`) or `int.MaxValue` (`NeverCompile`). Each call decrements the counter; the call that reads 0 (the 33rd) queues JIT compilation of the whole lambda on a thread-pool thread and is itself still interpreted; once the compiled delegate exists, every later call uses it. A lambda is **never interpreted** (JIT-compiled at once, synchronously) when the light compiler meets a method call with a by-ref parameter, an instance call on a non-primitive value type, or a constructor with a by-ref parameter; and trap-handler lambdas and single-expression lambdas are always built with `LambdaExpression.Compile()` (never interpreted). Loops are promoted separately (CF-CMP-002).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L1974-L2077", + "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2096", + "src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052", + "src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246", + "src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs::LightDelegateCreator@L30-L220" + ], + "owner": "02-language/TIERING-POLICY.md", + "status": "open" + }, + { + "id": "CF-CMP-002", + "subject": "every threshold and limit of the tiering policy", + "statement": "**Tiering thresholds and limits.** Statement count above which a script block is never JIT-compiled by call count: 300 (`_stmtCount > 300`, `Compiler.Compile`). Interpreted calls before JIT: 32 (`LightCompiler.DefaultCompilationThreshold`), so JIT starts on the 33rd call; `int.MaxValue` for `NeverCompile`; threshold 0 or 1 means synchronous compilation (never produced by `Compiler`). Loop iterations before loop JIT: 16 (constructor argument of `EnterLoopInstruction` in `EnterLoopExpression.AddInstructions`), so the loop JIT starts on the 17th arrival at the loop top. Loop statement count at or above which a loop is never promoted: 300 (`LoopStatementCount < 300` required to emit the loop counter). Maximum delegate parameters of an interpreted lambda: 16 (`LightLambda.MaxParameters`). Compile helper cache of delegate `Run` factories: 100 entries (`CacheDict` in `LightLambda`). Parsed-script cache (shares compiled delegates between clones): cleared completely when it holds more than 1024 entries (`ScriptBlock.CacheScriptBlock`).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7021", + "src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052", + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177" + ], + "owner": "02-language/TIERING-POLICY.md", + "status": "open" + }, + { + "id": "CF-CMP-003", + "subject": "optimized-versus-unoptimized preconditions", + "statement": "**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used for script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83", + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177", + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552", + "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847" + ], + "owner": "02-language/TIERING-POLICY.md", + "status": "open" + } + ], + "deviations": [ + { + "id": "DEV-TIER-060", + "baseline_behavior": "Hot loops are promoted from interpreter to JIT in the background mid-execution via in-place instruction replacement.", + "proposed_behavior": "No mid-execution promotion; any optimizer runs synchronously at defined points.", + "benefit": "Deterministic execution, simpler frames.", + "compatibility_cost": "Long top-level loops lose the speed-up PowerShell gets.", + "affected_cfs": [ + "CF-CMP-002" + ], + "status": "PROPOSED" + }, + { + "id": "DEV-EQV-061", + "baseline_behavior": "Interpreter keeps typed arithmetic/comparison instruction families inherited from the DLR.", + "proposed_behavior": "Do not port them; numeric semantics come from the evaluator's conversion rules.", + "benefit": "Removes dead and defective code.", + "compatibility_cost": "None observable (instructions unreachable from PowerShell IR).", + "affected_cfs": [], + "status": "PROPOSED" + } + ] +} \ No newline at end of file diff --git a/docs-out/.work/claims/T21.jsonl b/docs-out/.work/claims/T21.jsonl new file mode 100644 index 0000000..41e3d46 --- /dev/null +++ b/docs-out/.work/claims/T21.jsonl @@ -0,0 +1,19 @@ +{"id": "CLM-LEX-0001", "subject_key": "tokenizer.mode.initial", "predicate": "initial mode after Initialize", "value": "Command", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-001"} +{"id": "CLM-LEX-0002", "subject_key": "tokenizer.mode.keywords_recognized_in", "predicate": "keywords recognized only in modes", "value": "Command,Signature (and not WantSimpleName)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanIdentifier@L4402-L4458"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0003", "subject_key": "tokenizer.mode.dash_operators_recognized_in", "predicate": "-eq/-and style operators produced only in mode", "value": "Expression", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanParameter@L3185-L3338"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-001"} +{"id": "CLM-LEX-0004", "subject_key": "tokenizer.mode.switch_rescans_lookahead", "predicate": "mode switch rescans peeked token unless ParseModeInvariant", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-001"} +{"id": "CLM-LEX-0005", "subject_key": "tokenizer.lookahead.depth", "predicate": "parser token lookahead", "value": "1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser@L495-L497"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0006", "subject_key": "tokenizer.numeric.multiplier_requires_b", "predicate": "multiplier suffix k/m/g/t/p must be followed by b", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0007", "subject_key": "tokenizer.numeric.multiplier_base", "predicate": "multiplier base", "value": "1024", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0008", "subject_key": "tokenizer.numeric.type_suffixes", "predicate": "single type suffix letters", "value": "u s l d y n", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0009", "subject_key": "tokenizer.numeric.signed_requires_flag", "predicate": "leading sign scanned as part of a number only when AllowSignedNumbers", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0010", "subject_key": "tokenizer.numeric.default_promotion", "predicate": "no-suffix integer promotion", "value": "int,long,decimal(decimal format only),double", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0011", "subject_key": "tokenizer.string.single_quote_escape", "predicate": "backtick escapes in single-quoted strings", "value": "none", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanStringLiteral@L2317-L2360"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0012", "subject_key": "tokenizer.herestring.footer", "predicate": "here-string footer position", "value": "quote+@ at start of a line", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanHereStringLiteral@L2685-L2753"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0013", "subject_key": "tokenizer.subexpr.prescan", "predicate": "$() in strings pre-scanned by counting parentheses only", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanSubExpression@L2362-L2447"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0014", "subject_key": "tokenizer.line_endings", "predicate": "characters that end a line for position tracking", "value": "CR, LF, CRLF", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0015", "subject_key": "tokenizer.comments.block_nest", "predicate": "block comments nest", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanBlockComment@L1809-L1837"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0016", "subject_key": "tokenizer.redirection.stream_digits", "predicate": "digits forming stream-numbered redirection", "value": "1-6", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0017", "subject_key": "tokenizer.chain_operators.mode_dependent", "predicate": "&& and || tokenized in every mode", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0018", "subject_key": "tokenizer.questionmark.expression_only", "predicate": "?, ??, ??= tokens produced only in Expression mode", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0019", "subject_key": "tokenizer.requires.recognition", "predicate": "#requires must follow '#' immediately", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanToEndOfCommentLine@L1605-L1754"], "doc": "LANG-TOKENIZER-001", "cf": null} diff --git a/docs-out/.work/claims/T22.jsonl b/docs-out/.work/claims/T22.jsonl new file mode 100644 index 0000000..afb8e64 --- /dev/null +++ b/docs-out/.work/claims/T22.jsonl @@ -0,0 +1,17 @@ +{"id": "CLM-GRM-0001", "subject_key": "parser.lookahead.tokens", "predicate": "maximum pushed-back tokens", "value": "1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser@L495-L497"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0002", "subject_key": "parser.binary_operators.associativity", "predicate": "associativity of all binary operators", "value": "left", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0003", "subject_key": "parser.comma.vs_binary", "predicate": "comma operator binds tighter than binary operators (operands are ArrayLiteralRule)", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0004", "subject_key": "parser.statement.start", "predicate": "statement start tries expression in Expression mode before command", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0005", "subject_key": "parser.statement.terminator_after_block", "predicate": "terminator required after statement ending in closing brace", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0006", "subject_key": "parser.assignment.rhs", "predicate": "right side of an assignment is parsed by", "value": "StatementRule", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0007", "subject_key": "parser.hashtable.value", "predicate": "hashtable entry value is parsed by", "value": "StatementRule in Command mode", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetKeyValuePair@L7405-L7491"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0008", "subject_key": "parser.chain_operators.associativity", "predicate": "&& and || chaining", "value": "left nested PipelineChainAst", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0009", "subject_key": "parser.errors.dedup_key", "predicate": "error de-duplication key", "value": "same ErrorId and same start/end line and column", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057"], "doc": "LANG-PARSER-001", "cf": "CF-ERR-002"} +{"id": "CLM-GRM-0010", "subject_key": "parser.errors.max_count", "predicate": "limit on number of parse errors", "value": "none", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057"], "doc": "LANG-PARSER-001", "cf": "CF-ERR-002"} +{"id": "CLM-GRM-0011", "subject_key": "parser.stack.retry", "predicate": "stack exhaustion handling", "value": "retry once on a new thread then ScriptTooComplicated", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227"], "doc": "LANG-PARSER-001", "cf": "CF-ERR-002"} +{"id": "CLM-GRM-0012", "subject_key": "parser.array_type.max_dimensions", "predicate": "maximum array type rank", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.CompleteArrayTypeName@L1523-L1627"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0013", "subject_key": "parser.using_assembly.resolution_time", "predicate": "using assembly path resolved at", "value": "parse time", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0014", "subject_key": "parser.class.constrained_language", "predicate": "class keyword in Constrained Language Mode", "value": "rejected unless lockdown policy is Audit", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ClassDefinitionRule@L4208-L4379"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0015", "subject_key": "parser.mode_analysis.post_parse_checks", "predicate": "PerformPostParseChecks runs", "value": "only when ParseMode is Default", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0016", "subject_key": "parser.switch.flags_matching", "predicate": "switch flags -regex -wildcard -exact -casesensitive -parallel -file matched by", "value": "case-insensitive prefix", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SwitchStatementRule@L2543-L2891"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0017", "subject_key": "parser.member_access.whitespace", "predicate": "whitespace allowed before member access operator or invocation paren", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.MemberAccessRule@L7718-L7775"], "doc": "LANG-PARSER-001", "cf": null} diff --git a/docs-out/.work/claims/T23.jsonl b/docs-out/.work/claims/T23.jsonl new file mode 100644 index 0000000..69ae374 --- /dev/null +++ b/docs-out/.work/claims/T23.jsonl @@ -0,0 +1,13 @@ +{"id": "CLM-AST-0001", "subject_key": "ast.parent.set_once", "predicate": "a node can have a parent assigned more than once", "value": "false (AstIsReused)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0002", "subject_key": "ast.visit.pre_order_post_hook", "predicate": "PostVisit is called on every exit path (Continue, SkipChildren, StopVisit)", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0003", "subject_key": "ast.visit.v2_nodes", "predicate": "nodes visited only by second-generation visitors", "value": "TypeDefinition, PropertyMember, FunctionMember, BaseCtorInvokeMemberExpression, UsingStatement, ConfigurationDefinition, DynamicKeywordStatement, TernaryExpression, PipelineChain", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0004", "subject_key": "ast.find.root_tested", "predicate": "Find/FindAll test the root node itself", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/AstVisitor.cs::AstSearcher@L404-L617"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0005", "subject_key": "ast.find.nested_skip", "predicate": "nested scopes skipped when searchNestedScriptBlocks is false", "value": "FunctionDefinition, Trap, ScriptBlockExpression (not ScriptBlockAst)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/AstVisitor.cs::AstSearcher@L404-L617"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0006", "subject_key": "ast.safevalue.max_visits", "predicate": "safe-value visit cap", "value": "5000", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0007", "subject_key": "ast.safevalue.max_hashtable_keys", "predicate": "safe-value hashtable key cap", "value": "500", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0008", "subject_key": "ast.safevalue.binary_expression", "predicate": "binary expressions are safe values", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0009", "subject_key": "ast.constant.evaluation", "predicate": "constant folding value computed by", "value": "compiling and invoking the expression", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ConstantValues.cs::ConstantValueVisitor@L332-L577"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0010", "subject_key": "ast.hashtable.value_type", "predicate": "hashtable entry value node type", "value": "StatementAst", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::HashtableAst@L10128-L10218"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0011", "subject_key": "ast.assignment.rhs_normalization", "predicate": "single-command-expression pipeline on the right of an assignment is replaced by the expression", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::AssignmentStatementAst@L6379-L6495"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0012", "subject_key": "ast.pipelinechain.shape", "predicate": "PipelineChainAst right side", "value": "always PipelineAst (left nested)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::PipelineChainAst@L5549-L5647"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0013", "subject_key": "ast.ternary.visitor_gate", "predicate": "TernaryExpressionAst visited by v1 visitor", "value": "children yes, VisitTernaryExpression no", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::TernaryExpressionAst@L7404-L7497"], "doc": "LANG-AST-001", "cf": null} diff --git a/docs-out/.work/claims/T24.jsonl b/docs-out/.work/claims/T24.jsonl new file mode 100644 index 0000000..2ed5458 --- /dev/null +++ b/docs-out/.work/claims/T24.jsonl @@ -0,0 +1,9 @@ +{"id": "CLM-SEM-0001", "subject_key": "semantic.order", "predicate": "post-parse analysis order", "value": "SymbolResolver then SemanticChecks (ParseMode.Default only)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst.PerformPostParseChecks@L1416-L1440"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0002", "subject_key": "semantic.unresolved_type_error", "predicate": "unresolved type outside a type definition", "value": "no parse error (late bound)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SymbolResolver.cs::SymbolResolver@L275-L729"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0003", "subject_key": "semantic.array_type.max_depth", "predicate": "array type name nesting limit", "value": "200 (ScriptTooComplicated)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0004", "subject_key": "variableanalysis.force_dynamic", "predicate": "slot sentinel for dynamic variable", "value": "-2", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0005", "subject_key": "variableanalysis.unanalyzed", "predicate": "slot sentinel for unanalyzed variable", "value": "-1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0006", "subject_key": "variableanalysis.pessimizing_cmdlets", "predicate": "commands that disable slot allocation", "value": "New-Variable, Remove-Variable, Set-Variable, Set-PSBreakpoint and aliases", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/VariableAnalysis.cs::FindAllVariablesVisitor@L44-L323"], "doc": "LANG-SEMANTIC-001", "cf": "CF-CMP-003"} +{"id": "CLM-SEM-0007", "subject_key": "typeresolver.order", "predicate": "type name resolution order", "value": "session scope types, builtin accelerators, assemblies, user accelerators, then using-namespace retries", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolver@L31-L506"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0008", "subject_key": "restricted.default_variables", "predicate": "variables allowed in data sections by default", "value": "PSCulture, PSUICulture, true, false, null", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0009", "subject_key": "typedefiner.run_time", "predicate": "class and enum definition errors reported at", "value": "compile of the root script block", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505"], "doc": "LANG-SEMANTIC-001", "cf": null} diff --git a/docs-out/.work/claims/T25.jsonl b/docs-out/.work/claims/T25.jsonl new file mode 100644 index 0000000..81965a1 --- /dev/null +++ b/docs-out/.work/claims/T25.jsonl @@ -0,0 +1,9 @@ +{"id": "CLM-ENG-0001", "subject_key": "compiler.visit_method_count", "predicate": "Visit methods in Compiler", "value": "63", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0002", "subject_key": "compiler.statement_dispatch", "predicate": "statement lists with several statements use", "value": "integer dispatch with goto and per-statement catch", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0003", "subject_key": "compiler.return_semantics", "predicate": "return writes the value to", "value": "the output pipe (class methods: return pipe)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitReturnStatement@L5573-L5627"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0004", "subject_key": "compiler.switch.clauses", "predicate": "switch tests", "value": "every clause for every item; default only when none matched", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitSwitchStatement@L4410-L4542"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0005", "subject_key": "compiler.foreach.enumeration", "predicate": "foreach enumerator creation", "value": "PSEnumerableBinder; non-enumerable wrapped in one-element array", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateIteratorStatement@L4931-L5056"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0006", "subject_key": "compiler.loop.nonlocal_break", "predicate": "non-local break/continue", "value": "BreakException/ContinueException caught by enclosing loop", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateLoopBreakContinueCatchBlocks@L4731-L4758"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0007", "subject_key": "compiler.scriptblock.cache_cap", "predicate": "parsed script block cache cap", "value": "1024 (cleared completely when exceeded)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177"], "doc": "LANG-ENGINE-001", "cf": "CF-CMP-002"} +{"id": "CLM-ENG-0008", "subject_key": "compiler.single_expression.tier", "predicate": "single-expression and trap lambdas", "value": "always JIT compiled", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleExpression@L2204-L2227"], "doc": "LANG-ENGINE-001", "cf": "CF-CMP-001"} +{"id": "CLM-ENG-0009", "subject_key": "compiler.log_segment", "predicate": "script block log segment size when text length >= 20000", "value": "10000 plus random 0..9999", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177"], "doc": "LANG-ENGINE-001", "cf": null} diff --git a/docs-out/.work/claims/T27.jsonl b/docs-out/.work/claims/T27.jsonl new file mode 100644 index 0000000..810a50c --- /dev/null +++ b/docs-out/.work/claims/T27.jsonl @@ -0,0 +1,10 @@ +{"id": "CLM-TIER-0001", "subject_key": "tiering.stmt_limit", "predicate": "statement count above which NeverCompile is chosen", "value": "300", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L1974-L2077"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0002", "subject_key": "tiering.call_threshold", "predicate": "interpreted calls before JIT is queued (DefaultCompilationThreshold)", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0003", "subject_key": "tiering.loop_iterations", "predicate": "loop iterations before loop JIT", "value": "16", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7021"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0004", "subject_key": "tiering.loop_stmt_limit", "predicate": "loop statement count limit for promotion", "value": "300", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7021"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0005", "subject_key": "tiering.always_compile_used", "predicate": "AlwaysCompile passed by any caller", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2096"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-001"} +{"id": "CLM-TIER-0006", "subject_key": "tiering.optimized_condition", "predicate": "optimized variant requires", "value": "debugging off and new local scope", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-003"} +{"id": "CLM-TIER-0007", "subject_key": "tiering.allscope_fallback", "predicate": "optimized variant replaced by unoptimized when", "value": "an allocated variable could be AllScope", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-003"} +{"id": "CLM-TIER-0008", "subject_key": "tiering.forced_jit", "predicate": "forced immediate JIT", "value": "by-ref call/ctor or instance call on non-primitive value type", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-001"} +{"id": "CLM-TIER-0009", "subject_key": "tiering.promotion_async", "predicate": "JIT promotion runs on", "value": "thread pool unless threshold <= 1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-001"} +{"id": "CLM-TIER-0010", "subject_key": "interpreter.max_parameters", "predicate": "maximum parameters of an interpreted lambda", "value": "16", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightLambda.Generated.cs::LightLambda@L27-L28"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} diff --git a/docs-out/.work/files-ledger.csv b/docs-out/.work/files-ledger.csv index 14335db..963e495 100644 --- a/docs-out/.work/files-ledger.csv +++ b/docs-out/.work/files-ledger.csv @@ -1921,68 +1921,68 @@ src/System.Management.Automation/engine/hostifaces/RunspacePool.cs,1388,B,,,unre src/System.Management.Automation/engine/hostifaces/RunspacePoolInternal.cs,1669,B,,,unread src/System.Management.Automation/engine/hostifaces/internalHostuserInterfacesecurity.cs,74,B,,,unread src/System.Management.Automation/engine/hostifaces/pipelinebase.cs,1072,B,,,unread -src/System.Management.Automation/engine/interpreter/AddInstruction.cs,285,A,,,unread -src/System.Management.Automation/engine/interpreter/ArrayOperations.cs,133,A,,,unread -src/System.Management.Automation/engine/interpreter/BranchLabel.cs,129,A,,,unread -src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs,959,A,,,unread -src/System.Management.Automation/engine/interpreter/CallInstruction.cs,368,A,,,unread -src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs,795,A,,,unread -src/System.Management.Automation/engine/interpreter/DivInstruction.cs,152,A,,,unread -src/System.Management.Automation/engine/interpreter/DynamicInstructionN.cs,78,A,,,unread -src/System.Management.Automation/engine/interpreter/DynamicInstructions.Generated.cs,522,A,,,unread -src/System.Management.Automation/engine/interpreter/DynamicSplatInstruction.cs,53,A,,,unread -src/System.Management.Automation/engine/interpreter/EqualInstruction.cs,190,A,,,unread -src/System.Management.Automation/engine/interpreter/FieldOperations.cs,105,A,,,unread -src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs,169,A,,,unread -src/System.Management.Automation/engine/interpreter/ILightCallSiteBinder.cs,28,A,,,unread -src/System.Management.Automation/engine/interpreter/Instruction.cs,88,A,,,unread -src/System.Management.Automation/engine/interpreter/InstructionFactory.cs,130,A,,,unread -src/System.Management.Automation/engine/interpreter/InstructionList.cs,1221,A,,,unread -src/System.Management.Automation/engine/interpreter/InterpretedFrame.cs,333,A,,,unread -src/System.Management.Automation/engine/interpreter/Interpreter.cs,110,A,,,unread -src/System.Management.Automation/engine/interpreter/LabelInfo.cs,364,A,,,unread -src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs,169,A,,,unread -src/System.Management.Automation/engine/interpreter/LightCompiler.cs,2052,A,,,unread -src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs,220,A,,,unread -src/System.Management.Automation/engine/interpreter/LightLambda.Generated.cs,769,A,,,unread -src/System.Management.Automation/engine/interpreter/LightLambda.cs,303,A,,,unread -src/System.Management.Automation/engine/interpreter/LightLambdaClosureVisitor.cs,304,A,,,unread -src/System.Management.Automation/engine/interpreter/LocalAccess.cs,456,A,,,unread -src/System.Management.Automation/engine/interpreter/LocalVariables.cs,322,A,,,unread -src/System.Management.Automation/engine/interpreter/LoopCompiler.cs,385,A,,,unread -src/System.Management.Automation/engine/interpreter/MulInstruction.cs,285,A,,,unread -src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs,190,A,,,unread -src/System.Management.Automation/engine/interpreter/NumericConvertInstruction.cs,285,A,,,unread -src/System.Management.Automation/engine/interpreter/PowerShellInstructions.cs,51,A,,,unread -src/System.Management.Automation/engine/interpreter/RuntimeVariables.cs,55,A,,,unread -src/System.Management.Automation/engine/interpreter/StackOperations.cs,113,A,,,unread -src/System.Management.Automation/engine/interpreter/SubInstruction.cs,285,A,,,unread -src/System.Management.Automation/engine/interpreter/TypeOperations.cs,197,A,,,unread -src/System.Management.Automation/engine/interpreter/Utilities.cs,1157,A,,,unread +src/System.Management.Automation/engine/interpreter/AddInstruction.cs,285,A,1-285,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/ArrayOperations.cs,133,A,1-133,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/BranchLabel.cs,129,A,1-129,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs,959,A,1-959,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/CallInstruction.cs,368,A,1-368,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs,795,A,1-795,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/DivInstruction.cs,152,A,1-152,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/DynamicInstructionN.cs,78,A,1-78,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/DynamicInstructions.Generated.cs,522,A,1-522,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/DynamicSplatInstruction.cs,53,A,1-53,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/EqualInstruction.cs,190,A,1-190,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/FieldOperations.cs,105,A,1-105,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs,169,A,1-169,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/ILightCallSiteBinder.cs,28,A,1-28,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/Instruction.cs,88,A,1-88,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/InstructionFactory.cs,130,A,1-130,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/InstructionList.cs,1221,A,1-1221,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/InterpretedFrame.cs,333,A,1-333,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/Interpreter.cs,110,A,1-110,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LabelInfo.cs,364,A,1-364,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs,169,A,1-169,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LightCompiler.cs,2052,A,1-2052,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs,220,A,1-220,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LightLambda.Generated.cs,769,A,1-769,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LightLambda.cs,303,A,1-303,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LightLambdaClosureVisitor.cs,304,A,1-304,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LocalAccess.cs,456,A,1-456,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LocalVariables.cs,322,A,1-322,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/LoopCompiler.cs,385,A,1-385,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/MulInstruction.cs,285,A,1-285,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs,190,A,1-190,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/NumericConvertInstruction.cs,285,A,1-285,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/PowerShellInstructions.cs,51,A,1-51,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/RuntimeVariables.cs,55,A,1-55,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/StackOperations.cs,113,A,1-113,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/SubInstruction.cs,285,A,1-285,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/TypeOperations.cs,197,A,1-197,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read +src/System.Management.Automation/engine/interpreter/Utilities.cs,1157,A,1-1157,LANG-EQUIV-001;LANG-INSTR-001;LANG-TIERING-001,read src/System.Management.Automation/engine/lang/codegen.cs,110,A,,,unread -src/System.Management.Automation/engine/lang/parserutils.cs,2010,A,,,unread +src/System.Management.Automation/engine/lang/parserutils.cs,2010,A,1-2010,LANG-ENGINE-001,read src/System.Management.Automation/engine/lang/scriptblock.cs,1479,A,,,unread src/System.Management.Automation/engine/lang/interface/PSParseError.cs,52,A,,,unread src/System.Management.Automation/engine/lang/interface/PSParser.cs,194,A,,,unread src/System.Management.Automation/engine/lang/interface/PSToken.cs,644,A,,,unread -src/System.Management.Automation/engine/parser/AstVisitor.cs,822,A,,,unread -src/System.Management.Automation/engine/parser/CharTraits.cs,414,A,,,unread -src/System.Management.Automation/engine/parser/Compiler.cs,7082,A,,,unread -src/System.Management.Automation/engine/parser/ConstantValues.cs,578,A,,,unread +src/System.Management.Automation/engine/parser/AstVisitor.cs,822,A,1-822,LANG-AST-001;LANG-ASTCAT-001,read +src/System.Management.Automation/engine/parser/CharTraits.cs,414,A,1-414,LANG-TOKCAT-001;LANG-TOKENIZER-001,read +src/System.Management.Automation/engine/parser/Compiler.cs,7082,A,1-7082,LANG-COMPMAP-001;LANG-ENGINE-001;LANG-IR-001,read +src/System.Management.Automation/engine/parser/ConstantValues.cs,578,A,1-578,LANG-AST-001,read src/System.Management.Automation/engine/parser/DebugViewWriter.cs,1219,A,,,unread -src/System.Management.Automation/engine/parser/PSType.cs,1520,A,,,unread -src/System.Management.Automation/engine/parser/Parser.cs,8279,A,,,unread -src/System.Management.Automation/engine/parser/Position.cs,816,A,,,unread -src/System.Management.Automation/engine/parser/PreOrderVisitor.cs,264,A,,,unread -src/System.Management.Automation/engine/parser/SafeValues.cs,807,A,,,unread -src/System.Management.Automation/engine/parser/SemanticChecks.cs,2455,A,,,unread -src/System.Management.Automation/engine/parser/SymbolResolver.cs,763,A,,,unread +src/System.Management.Automation/engine/parser/PSType.cs,1520,A,1-1520,LANG-SEMANTIC-001,read +src/System.Management.Automation/engine/parser/Parser.cs,8279,A,1-8279,LANG-ERRCAT-001;LANG-GRAMMAR-001;LANG-PARSER-001,read +src/System.Management.Automation/engine/parser/Position.cs,816,A,1-816,LANG-AST-001,read +src/System.Management.Automation/engine/parser/PreOrderVisitor.cs,264,A,1-264,LANG-AST-001;LANG-ASTCAT-001,read +src/System.Management.Automation/engine/parser/SafeValues.cs,807,A,1-807,LANG-AST-001,read +src/System.Management.Automation/engine/parser/SemanticChecks.cs,2455,A,1-2455,LANG-SEMANTIC-001,read +src/System.Management.Automation/engine/parser/SymbolResolver.cs,763,A,1-763,LANG-SEMANTIC-001,read src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs,3338,A,,,unread -src/System.Management.Automation/engine/parser/TypeResolver.cs,980,A,,,unread -src/System.Management.Automation/engine/parser/VariableAnalysis.cs,1848,A,,,unread -src/System.Management.Automation/engine/parser/ast.cs,10845,A,,,unread -src/System.Management.Automation/engine/parser/token.cs,1575,A,,,unread -src/System.Management.Automation/engine/parser/tokenizer.cs,5083,A,,,unread +src/System.Management.Automation/engine/parser/TypeResolver.cs,980,A,1-980,LANG-SEMANTIC-001,read +src/System.Management.Automation/engine/parser/VariableAnalysis.cs,1848,A,1-1848,LANG-SEMANTIC-001,read +src/System.Management.Automation/engine/parser/ast.cs,10845,A,1-10845,LANG-AST-001;LANG-ASTCAT-001,read +src/System.Management.Automation/engine/parser/token.cs,1575,A,1-1575,LANG-TOKCAT-001;LANG-TOKENIZER-001,read +src/System.Management.Automation/engine/parser/tokenizer.cs,5083,A,1-5083,LANG-TOKCAT-001;LANG-TOKENIZER-001,read src/System.Management.Automation/engine/remoting/client/ClientMethodExecutor.cs,243,B,,,unread src/System.Management.Automation/engine/remoting/client/ClientRemotePowerShell.cs,1000,B,,,unread src/System.Management.Automation/engine/remoting/client/Job.cs,4524,B,,,unread @@ -2079,7 +2079,7 @@ src/System.Management.Automation/engine/remoting/server/serverremotesession.cs,1 src/System.Management.Automation/engine/remoting/server/serverremotesessionstatemachine.cs,1023,B,,,unread src/System.Management.Automation/engine/remoting/server/serverremotingprotocol.cs,100,B,,,unread src/System.Management.Automation/engine/remoting/server/serverremotingprotocolimplementation.cs,244,B,,,unread -src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs,2553,A,,,unread +src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs,2553,A,1-2553,LANG-ENGINE-001,read src/System.Management.Automation/engine/runtime/MutableTuple.cs,2536,A,,,unread src/System.Management.Automation/engine/runtime/ScriptBlockToPowerShell.cs,953,A,,,unread src/System.Management.Automation/engine/runtime/Binding/Binders.cs,8015,A,,,unread diff --git a/docs-out/.work/ledger-proposals/T21.csv b/docs-out/.work/ledger-proposals/T21.csv new file mode 100644 index 0000000..3ce771c --- /dev/null +++ b/docs-out/.work/ledger-proposals/T21.csv @@ -0,0 +1,4 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/parser/tokenizer.cs,1-5083,LANG-TOKENIZER-001;LANG-TOKCAT-001 +src/System.Management.Automation/engine/parser/token.cs,1-1575,LANG-TOKENIZER-001;LANG-TOKCAT-001 +src/System.Management.Automation/engine/parser/CharTraits.cs,1-414,LANG-TOKENIZER-001;LANG-TOKCAT-001 diff --git a/docs-out/.work/ledger-proposals/T22.csv b/docs-out/.work/ledger-proposals/T22.csv new file mode 100644 index 0000000..ef70e16 --- /dev/null +++ b/docs-out/.work/ledger-proposals/T22.csv @@ -0,0 +1,2 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/parser/Parser.cs,1-8279,LANG-PARSER-001;LANG-GRAMMAR-001;LANG-ERRCAT-001 diff --git a/docs-out/.work/ledger-proposals/T23.csv b/docs-out/.work/ledger-proposals/T23.csv new file mode 100644 index 0000000..094c19a --- /dev/null +++ b/docs-out/.work/ledger-proposals/T23.csv @@ -0,0 +1,7 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/parser/ast.cs,1-10845,LANG-AST-001;LANG-ASTCAT-001 +src/System.Management.Automation/engine/parser/AstVisitor.cs,1-822,LANG-AST-001;LANG-ASTCAT-001 +src/System.Management.Automation/engine/parser/PreOrderVisitor.cs,1-264,LANG-AST-001;LANG-ASTCAT-001 +src/System.Management.Automation/engine/parser/ConstantValues.cs,1-578,LANG-AST-001 +src/System.Management.Automation/engine/parser/SafeValues.cs,1-807,LANG-AST-001 +src/System.Management.Automation/engine/parser/Position.cs,1-816,LANG-AST-001 diff --git a/docs-out/.work/ledger-proposals/T24.csv b/docs-out/.work/ledger-proposals/T24.csv new file mode 100644 index 0000000..3734520 --- /dev/null +++ b/docs-out/.work/ledger-proposals/T24.csv @@ -0,0 +1,6 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/parser/SemanticChecks.cs,1-2455,LANG-SEMANTIC-001 +src/System.Management.Automation/engine/parser/SymbolResolver.cs,1-763,LANG-SEMANTIC-001 +src/System.Management.Automation/engine/parser/TypeResolver.cs,1-980,LANG-SEMANTIC-001 +src/System.Management.Automation/engine/parser/PSType.cs,1-1520,LANG-SEMANTIC-001 +src/System.Management.Automation/engine/parser/VariableAnalysis.cs,1-1848,LANG-SEMANTIC-001 diff --git a/docs-out/.work/ledger-proposals/T25.csv b/docs-out/.work/ledger-proposals/T25.csv new file mode 100644 index 0000000..3a9b872 --- /dev/null +++ b/docs-out/.work/ledger-proposals/T25.csv @@ -0,0 +1,4 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/parser/Compiler.cs,1-7082,LANG-ENGINE-001;LANG-COMPMAP-001;LANG-IR-001 +src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs,1-2553,LANG-ENGINE-001 +src/System.Management.Automation/engine/lang/parserutils.cs,1-2010,LANG-ENGINE-001 diff --git a/docs-out/.work/ledger-proposals/T27.csv b/docs-out/.work/ledger-proposals/T27.csv new file mode 100644 index 0000000..982de2c --- /dev/null +++ b/docs-out/.work/ledger-proposals/T27.csv @@ -0,0 +1,39 @@ +path,ranges_read,doc_ids +src/System.Management.Automation/engine/interpreter/AddInstruction.cs,1-285,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/ArrayOperations.cs,1-133,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/BranchLabel.cs,1-129,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs,1-959,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/CallInstruction.cs,1-368,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs,1-795,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/DivInstruction.cs,1-152,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/DynamicInstructionN.cs,1-78,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/DynamicInstructions.Generated.cs,1-522,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/DynamicSplatInstruction.cs,1-53,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/EqualInstruction.cs,1-190,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/FieldOperations.cs,1-105,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs,1-169,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/ILightCallSiteBinder.cs,1-28,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/Instruction.cs,1-88,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/InstructionFactory.cs,1-130,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/InstructionList.cs,1-1221,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/InterpretedFrame.cs,1-333,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/Interpreter.cs,1-110,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LabelInfo.cs,1-364,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs,1-169,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LightCompiler.cs,1-2052,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs,1-220,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LightLambda.Generated.cs,1-769,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LightLambda.cs,1-303,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LightLambdaClosureVisitor.cs,1-304,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LocalAccess.cs,1-456,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LocalVariables.cs,1-322,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/LoopCompiler.cs,1-385,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/MulInstruction.cs,1-285,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs,1-190,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/NumericConvertInstruction.cs,1-285,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/PowerShellInstructions.cs,1-51,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/RuntimeVariables.cs,1-55,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/StackOperations.cs,1-113,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/SubInstruction.cs,1-285,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/TypeOperations.cs,1-197,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 +src/System.Management.Automation/engine/interpreter/Utilities.cs,1-1157,LANG-INSTR-001;LANG-TIERING-001;LANG-EQUIV-001 diff --git a/docs-out/.work/numbers-proposals/T21.json b/docs-out/.work/numbers-proposals/T21.json new file mode 100644 index 0000000..abe58b0 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T21.json @@ -0,0 +1,2 @@ +[{"key":"TOKENIZER_KEYWORD_ROWS","cmd":"python3 docs-out/tools/gen_T21.py --count keywords","dataset":"tables.json","label":"APPROX","notes":"rows of Tokenizer.s_keywordText (regex-parsed array literal)"}, + {"key":"TOKENIZER_DASH_OPERATOR_ROWS","cmd":"python3 docs-out/tools/gen_T21.py --count dash_operators","dataset":"tables.json","label":"APPROX","notes":"rows of Tokenizer._operatorText"}] diff --git a/docs-out/.work/numbers-proposals/T22.json b/docs-out/.work/numbers-proposals/T22.json new file mode 100644 index 0000000..a6d7331 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T22.json @@ -0,0 +1,30 @@ +[ + { + "key": "PARSER_METHODS", + "cmd": "python3 docs-out/tools/gen_T22.py --count parser_methods", + "dataset": "parser-methods.json", + "label": "APPROX", + "notes": "methods of Parser.cs per scanner" + }, + { + "key": "BINARY_OPERATORS", + "cmd": "python3 docs-out/tools/gen_T22.py --count binary_operators", + "dataset": "tables.json", + "label": "APPROX", + "notes": "TokenKinds with BinaryOperator flag in s_staticTokenFlags" + }, + { + "key": "PARSERSTRINGS_KEYS", + "cmd": "python3 docs-out/tools/gen_T22.py --count parserstrings_keys", + "dataset": "resources.json", + "label": "APPROX", + "notes": "keys of ParserStrings.resx" + }, + { + "key": "PARSERSTRINGS_KEYS_PARSER_TOKENIZER", + "cmd": "python3 docs-out/tools/gen_T22.py --count parserstrings_keys_parser_tokenizer", + "dataset": "resources.json", + "label": "APPROX", + "notes": "ParserStrings keys with a use site in Parser.cs or tokenizer.cs" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T23.json b/docs-out/.work/numbers-proposals/T23.json new file mode 100644 index 0000000..8ac16c5 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T23.json @@ -0,0 +1,30 @@ +[ + { + "key": "AST_CLASSES_ASTCS", + "cmd": "python3 docs-out/tools/gen_T23.py --count ast_classes", + "dataset": "ast-hierarchy.json", + "label": "APPROX", + "notes": "Ast-derived classes declared in ast.cs" + }, + { + "key": "AST_CLASSES_ABSTRACT", + "cmd": "python3 docs-out/tools/gen_T23.py --count abstract", + "dataset": "ast-hierarchy.json", + "label": "APPROX", + "notes": "abstract classes among them" + }, + { + "key": "ICUSTOMASTVISITOR_METHODS", + "cmd": "python3 docs-out/tools/gen_T23.py --count iv1", + "dataset": "visitors.json", + "label": "APPROX", + "notes": "Visit methods in ICustomAstVisitor" + }, + { + "key": "ICUSTOMASTVISITOR2_METHODS", + "cmd": "python3 docs-out/tools/gen_T23.py --count iv2", + "dataset": "visitors.json", + "label": "APPROX", + "notes": "Visit methods added by ICustomAstVisitor2" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T24.json b/docs-out/.work/numbers-proposals/T24.json new file mode 100644 index 0000000..7ab528a --- /dev/null +++ b/docs-out/.work/numbers-proposals/T24.json @@ -0,0 +1,16 @@ +[ + { + "key": "SEMANTIC_CHECK_SITES", + "cmd": "python3 docs-out/tools/gen_T24.py --count sites", + "dataset": "types.json", + "label": "APPROX", + "notes": "nameof(ParserStrings.X) sites in SemanticChecks.cs, SymbolResolver.cs, PSType.cs" + }, + { + "key": "SEMANTIC_CHECK_IDS", + "cmd": "python3 docs-out/tools/gen_T24.py --count ids", + "dataset": "types.json", + "label": "APPROX", + "notes": "distinct ids among them" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T25.json b/docs-out/.work/numbers-proposals/T25.json new file mode 100644 index 0000000..1c43772 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T25.json @@ -0,0 +1,16 @@ +[ + { + "key": "COMPILER_VISIT_METHODS", + "cmd": "python3 docs-out/tools/gen_T25.py --count visits", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "Visit* methods of Compiler" + }, + { + "key": "COMPILER_HELPER_METHODS", + "cmd": "python3 docs-out/tools/gen_T25.py --count helpers", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "non-Visit methods of Compiler" + } +] \ No newline at end of file diff --git a/docs-out/.work/numbers-proposals/T27.json b/docs-out/.work/numbers-proposals/T27.json new file mode 100644 index 0000000..a9f6b29 --- /dev/null +++ b/docs-out/.work/numbers-proposals/T27.json @@ -0,0 +1,23 @@ +[ + { + "key": "INSTRUCTION_CLASSES", + "cmd": "python3 docs-out/tools/gen_T27.py --count instructions", + "dataset": "instructions.json", + "label": "APPROX", + "notes": "classes deriving from Instruction found by scanner" + }, + { + "key": "INSTRUCTION_ABSTRACT", + "cmd": "python3 docs-out/tools/gen_T27.py --count abstract", + "dataset": "instructions.json", + "label": "APPROX", + "notes": "abstract ones" + }, + { + "key": "INSTRUCTION_FILES", + "cmd": "python3 docs-out/tools/gen_T27.py --count files", + "dataset": "instructions.json", + "label": "APPROX", + "notes": "files holding them" + } +] \ No newline at end of file diff --git a/docs-out/02-language/02.1-tokenizer.md b/docs-out/02-language/02.1-tokenizer.md new file mode 100644 index 0000000..dd225a4 --- /dev/null +++ b/docs-out/02-language/02.1-tokenizer.md @@ -0,0 +1,316 @@ +--- +id: LANG-TOKENIZER-001 +title: Tokenizer +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-LEX-001, CF-LEX-002] +sources: [src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/token.cs, src/System.Management.Automation/engine/parser/CharTraits.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/enums.json, data/tables.json, data/members-tierA.json, data/parser-methods.json] +--- + +# Tokenizer + +> STATIC-ONLY — behavior not run-verified. Statements are `FACT-SRC` (read in the cited source) unless marked `INFERENCE` or `UNVERIFIED`. Full catalogs (every token kind, flag, keyword, dash operator, character class) are in [TOKEN-CATALOG.md](TOKEN-CATALOG.md) and are `GENERATED`. + +## 1. Purpose and scope [SPEC] + +The tokenizer turns script text into `Token` objects on demand: the parser calls `NextToken()` one token at a time and tells the tokenizer, through a `Mode` property, how to read the next characters. There is **no pre-pass** that tokenizes the whole file: the same characters can become different tokens depending on the mode the parser requests. `FACT-SRC` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081`, `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`). + +Scope: character-level rules, mode handling, number/string/variable/comment lexical rules, position tracking, the pushback (`Resync`) mechanism, and the `#requires` pre-processing that lives in `tokenizer.cs`. Out of scope here: grammar (see [02.2-parser.md](02.2-parser.md), planned T22), the AST, and DSC dynamic-keyword semantics beyond how they change keyword recognition. + +Terminology: this study says *tokenize* (text → tokens), *parse* (tokens → AST), per [GLOSSARY](../GLOSSARY.md). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L594-L5083` | `internal class` | the tokenizer: ~5000 lines, whole file read in 16 windows | A | +| `src/System.Management.Automation/engine/parser/token.cs::TokenKind@L18-L595` | `public enum` | token kinds (see [TOKEN-CATALOG](TOKEN-CATALOG.md) §3) | A | +| `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752` | `[Flags] public enum` | static traits of a kind (precedence, keyword, operator class...) | A | +| `src/System.Management.Automation/engine/parser/token.cs::TokenTraits@L759-L1211` | static class | `s_staticTokenFlags` and `s_tokenText` tables, `GetTraits`, `Text`, `GetBinaryPrecedence` | A | +| `src/System.Management.Automation/engine/parser/token.cs::Token@L1216-L1280` | `public class` | base token: kind, flags, extent | A | +| `src/System.Management.Automation/engine/parser/CharTraits.cs::CharTraits@L28-L89` | `internal enum` | per-character class bits; 128-entry ASCII table at `s_traits` | A | +| `src/System.Management.Automation/engine/parser/CharTraits.cs::CharExtensions@L230-L413` | static class | `IsWhitespace`, `IsDash`, `IsVariableStart`, `ForceStartNewToken`, ... | A | +| `src/System.Management.Automation/engine/parser/tokenizer.cs::DynamicKeyword@L71-L357` | `public class` | thread-local table of DSC dynamic keywords consulted when recognizing keywords | A | +| `src/System.Management.Automation/engine/parser/tokenizer.cs::TokenizerState@L582-L593` | `internal class` | snapshot used by nested scans | A | + +`tokenizer.cs` also holds the `DynamicKeyword` family (L24-L488) and the `#requires` processor (L1892-L2275); neither is tokenizing proper (`FACT-SRC`, notes W1-W2, W7-W8). + +## 3. Responsibilities and non-responsibilities + +Responsible for: classifying characters (`CharTraits`), producing tokens in a mode, tracking token extents (including compensation for rewritten text), comment handling, signature-block detection, `#requires` extraction, number literal parsing to a typed value, pushback support. +Not responsible for: grammar decisions, newline significance (the tokenizer emits `NewLine` tokens; the parser decides), semantic checks, evaluating expandable strings (it only records nested tokens and a format string). + +## 4. Key types and contracts [IMPL] + +**Entry points** (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797`, `Tokenizer.NextToken@L4612-L5081`): `Initialize(fileName, input, tokenList)` builds the line-start table and sets `Mode = Command`; `NextToken()` returns the next `Token`; `GetRestorePoint()`/`Resync()` support backtracking (§6.6). + +**Token classes** (`src/System.Management.Automation/engine/parser/token.cs::Token@L1216-L1280` and subclasses, L1281-L1575): + +| Class | Extra data | Used for kinds | +|---|---|---| +| `Token` | `Kind`, `TokenFlags`, `Extent`, `Text` | punctuators, operators, keywords, `Identifier`, `NewLine`, `Comment`, `EndOfInput`, `Unknown` | +| `NumberToken` | `Value` (typed object) | `Number` | +| `ParameterToken` | `ParameterName`, `UsedColon` | `Parameter` | +| `VariableToken` | `VariablePath`, `Name` | `Variable`, `SplattedVariable` | +| `StringLiteralToken` / `StringExpandableToken` | `Value`; expandable adds `NestedTokens`, internal `FormatString` | `StringLiteral`, `StringExpandable`, `HereStringLiteral`, `HereStringExpandable`, and `Generic` | +| `LabelToken` | `LabelText` | `Label` | +| `InputRedirectionToken`, `MergingRedirectionToken` (`FromStream`, `ToStream`), `FileRedirectionToken` (`FromStream`, `Append`) | stream numbers | `Redirection` | +| `UnscannedSubExprToken` (internal) | `SkippedCharOffsets` (`BitArray`) | text of a `$(...)` inside a string, scanned later | + +`TokenKind` has 157 named members with explicit values 0..170; values 105-118 are 14 unnamed reserved slots (`FACT-SRC`, `src/System.Management.Automation/engine/parser/token.cs::TokenKind@L18-L595`, table placeholders `Reserved slot N`). The comment on L20-22 requires that adding a kind also updates `s_staticTokenFlags` and `s_tokenText`; a `#if DEBUG` static constructor asserts both tables have `(int)TokenKind.Clean + 1` entries (`src/System.Management.Automation/engine/parser/token.cs::TokenTraits@L1159-L1180`). + +**Mode** (`src/System.Management.Automation/engine/parser/tokenizer.cs::TokenizerMode@L489-L495`): `Command`, `Expression`, `TypeName`, `Signature` (class/method declaration). `Initialize` sets `Command`. Two further switches are properties of the tokenizer, not modes: `AllowSignedNumbers`, `ForceEndNumberOnTernaryOpChars`, and `WantSimpleName` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Mode@L729-L752`). + +## 5. Data model + +- State: `_script`, `_tokenStart`, `_currentIndex`, `_nestedTokensAdjustment`, `_skippedCharOffsets`, `_beginSignatureExtent`, `TokenList`, `FirstToken`, `LastToken`, `RequiresTokens` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L597-L621`). +- Character classes: 128-entry `s_traits` table with bits `IdentifierStart`, `MultiplierStart`, `TypeSuffix`, `Whitespace`, `Newline`, `HexDigit`, `Digit`, `VarNameFirst`, `ForceStartNewToken`, `ForceStartNewAssemblyNameSpecToken`, `ForceStartNewTokenAfterNumber` (`src/System.Management.Automation/engine/parser/CharTraits.cs::CharTraits@L28-L89`; generated list in [TOKEN-CATALOG](TOKEN-CATALOG.md) §7). Characters ≥ 128 fall back to `char.IsLetter`/`IsLetterOrDigit`/`IsSeparator` (`src/System.Management.Automation/engine/parser/CharTraits.cs::CharExtensions@L230-L413`). +- **Newline is not whitespace** in the character classes (comment L235-236) — `NextToken` returns a `NewLine` token for `\r`/`\n` (`FACT-SRC`). +- Quote characters: single-quote class = `'` U+2018 U+2019 U+201A U+201B; double-quote class includes U+201C U+201D U+201E; dash class = `-` U+2013 U+2014 U+2015 (`src/System.Management.Automation/engine/parser/CharTraits.cs::SpecialChars@L6-L25`, `CharExtensions.IsDash`, `IsSingleQuote`, `IsDoubleQuote`). + +## 6. Control flow + +### 6.1 Dispatcher + +`NextToken` is a single `switch` on the current character (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081`). Skeleton, verified in source: + +```mermaid +flowchart TD + A[NextToken: GetChar] --> B{char} + B -->|space tab FF VT NBSP NEL| A + B -->|quote class| S[ScanStringLiteral / ScanStringExpandable] + B -->|at sign| AT[AtCurly AtParen here-string or splatted variable] + B -->|hash| C[ScanLineComment then continue] + B -->|CR LF| N[NewLine token] + B -->|backtick| BT[line continuation, skip whitespace, or escape starting a generic token] + B -->|dash plus equals star slash percent| OP[operators and ScanParameter ScanNumber] + B -->|dollar| V[DollarParen or ScanVariable] + B -->|digit dot| NUM[ScanNumber or ScanDot] + B -->|letter underscore| ID[ScanIdentifier] + B -->|NUL| E[EndOfInput at EOF else generic] + B -->|other| G[ScanGenericToken] +``` + +Rules by first character (all `FACT-SRC`, `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081` unless stated): + +| First char | Result | +|---|---| +| whitespace (` `, `\t`, `\f`, `\v`, NBSP, NEL, any `IsWhitespace`) | skipped | +| `'` or smart single quote | single-quoted string (`ScanStringLiteral`) | +| `"` or smart double quote | expandable string (`ScanStringExpandable`) | +| `@` | `@{` → `AtCurly`; `@(` → `AtParen`; `@'` / `@"` → here-string; `@` + variable-start → splatted variable; else error `UnrecognizedToken` and an `Unknown` token | +| `#` | line comment, consumed and **not returned** (saved only in `TokenList`); `<#` is a block comment | +| `\n`, `\r`(`\n`) | `NewLine` token | +| `` ` `` | followed by CR/LF → `LineContinuation` consumed, scanning continues; followed by other whitespace → whitespace skipped; at EOF → `IncompleteString`-class error; otherwise an escape (`Backtick`) that starts a generic token | +| `$` | `$(` → `DollarParen`; else `ScanVariable` | +| `<` | `InputRedirectionToken` (`RedirectInStd`) | +| `>` | `>>` append / `>` file redirection (stream 1) | +| digit `1`-`6` followed by `>` | stream-numbered redirection (`2>`, `2>>`, `2>&1`); only streams 1-6 | +| other digits, `.` + digit | number (`ScanNumber`); `.` otherwise `ScanDot` | +| letter, `_`, non-ASCII letter | `ScanIdentifier` | +| `(` `)` `{` `}` `]` `,` `;` | punctuator tokens | +| `[` | in Command mode followed by a non-terminator → generic token starting with `[`; else `LBracket` | +| `&` / `|` | `&&` / `||` (`AndAnd`/`OrOr`) else `Ampersand` / `Pipe`, **in every mode** | +| `?` | only in Expression mode: `??=`, `??`, `?`; elsewhere falls to the default → generic (so `?` as an alias name works in commands) | +| `!` | see §6.4 | +| `:` | `::` → `ColonColon` (generic in Command mode when a non-terminator follows); single `:` in Command mode → `ScanLabel`, else `Colon` | +| `\0` | at end of script → `EndOfInput` (extent one past the end); NUL inside the script → generic | +| anything else (`^`, `~`, `\`, ...) | default → generic token | + +### 6.2 Modes: what changes + +Only these call sites select a mode (`FACT-SRC`; each `SetTokenizerMode` is paired with a `finally` restoring the previous mode): + +| Trigger (parser method) | Mode set | Why (`INFERENCE` from the surrounding grammar comment) | Evidence | +|---|---|---|---| +| start of every `Initialize` | `Command` | a script starts as statement/command text | `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797` | +| `CommandRule` | `Command` | arguments are barewords | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CommandRule@L6444-L6684` (L6482) | +| `GetSingleCommandArgument` | `Command` | one argument | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetSingleCommandArgument@L6266-L6283` | +| `ScriptBlockExpressionRule`, `SubExpressionRule`, `ParenthesizedExpressionRule` (body of `{ }`, `$( )`, `( )`) | `Command` | a nested block starts at statement level | L7505, L7536, L7584 | +| `ExpressionRule`, `BinaryExpressionRule` | `Expression` | operators, no barewords | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpressionRule@L6696-L6790` (L6712), `Parser.BinaryExpressionRule@L6798-L6950` (L6842) | +| first element of a pipeline / pipeline chain when no command is found | `Expression` | an expression can start a pipeline (`"x" \| %`) | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineRule@L5990-L6160` (L6022), `Parser.PipelineChainRule@L5773-L5988` (L5799) | +| `ParameterRule`, `AttributeRule`, `GetKeyValuePair` (value), `EnumMemberRule` | `Expression` | parameter defaults, attribute arguments, hashtable values | L970, L1094, L7416, L4848 | +| `TypeNameRule`, `NextTypeIdentifierToken`, `GenericMethodArgumentsRule` | `TypeName` | body of `[...]` type literals | L1312, L4684, L7800 | +| `ClassDefinitionRule`, `EnumDefinitionRule`, `MethodDeclarationRule` (header) | `Signature` | keywords such as `class`/`hidden` must be recognized | L4269, L4735, L5211 | +| `MethodDeclarationRule` (body) | `Command` | method bodies are statements | L5303 | + +Switching is **not free**: `SetTokenizerMode` first checks whether the already-peeked (`_ungotToken`) token's kind has the `ParseModeInvariant` trait; if it does not, the token is thrown away and the characters are rescanned in the new mode via `Resync` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`). `ParseModeInvariant` is set statically for `NewLine`, `LineContinuation`, `Comment`, `EndOfInput`, the four string kinds, `LParen RParen LCurly RCurly RBracket AtParen AtCurly DollarParen Semi AndAnd OrOr Ampersand Pipe Comma RedirectInStd` (`src/System.Management.Automation/engine/parser/token.cs::TokenTraits.s_staticTokenFlags@L759-L957`, notes). The parser supports one token of lookahead only (`Diagnostics.Assert(_ungotToken == null, "Only 1 token lookahead is supported")`, `src/System.Management.Automation/engine/parser/Parser.cs::Parser@L495-L497`). + +**Mode rule.** The tokenizer starts in Command mode; the parser switches to Expression mode when it parses an expression operand/operator position (including the first element of a pipeline when it is not a command), to Command mode when it parses a command's arguments or the body of `{ }`, `( )`, `$( )`, to TypeName mode inside `[...]` type literals, and to Signature mode for class/enum/method headers; every switch is restored in a `finally` block, and a switch discards and rescans a pending lookahead token unless its kind is `ParseModeInvariant`. + +### 6.3 What differs between Command and Expression mode + +(all `FACT-SRC` in `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer`) + +| Construct | Command mode | Expression mode | Evidence | +|---|---|---|---| +| `-name` after a dash | `ParameterToken`, even `-eq`, `-and`; digits and `-`/`_` stay in the name | dash operator if the text is in the operator table (`-eq`, `-and`, `-f`, ...), else `ParameterToken`; digits are not appended | `ScanParameter@L3185-L3338` | +| `-` followed by space/other | a lone `-` followed by a non-terminator (not a letter, `_`, `?`) starts a generic token (`a-b` is one word because the identifier scan hands over) | `Minus` | `CheckOperatorInCommandMode@L3340-L3380` | +| `+ = * / % ++ -- += -= ...` | if the next char is not a token terminator, becomes the start of a **generic token** | operator token | `CheckOperatorInCommandMode@L3340-L3380` | +| identifier then non-terminator (`foo-bar`, `a.b`) | generic token | identifier | `ScanIdentifier@L4402-L4458` | +| keywords (`if`, `foreach`, ...) | recognized (unless `WantSimpleName`) | **never** recognized; plain `Identifier` | `ScanIdentifier@L4402-L4458` | +| `[` | generic token if followed by a non-terminator | `LBracket` | `NextToken@L4612-L5081` | +| `.` | `.` followed by whitespace/NUL/CR/LF is the dot-source operator; `.\x`, `.foo` become generic; `..` is `DotDot` only when a terminator follows | `Dot` / `DotDot` | `ScanDot@L4367-L4400` | +| `?` | generic word | `?`, `??`, `??=` | `NextToken@L4612-L5081` | +| `!` | generic if a non-terminator follows | generic only if an identifier start follows; else `Exclaim` (digits probed as a number) | `NextToken@L4612-L5081` | +| `:` | `ScanLabel` (statement labels `:outer`) | `Colon` | `ScanLabel@L4574-L4610` | +| number followed by letters (`77z.exe`, `1k`) | not a number → generic | not a number → generic (or `BadNumericConstant` if the shape is numeric but the value overflows) | `ScanNumber@L3884-L3931` | +| variable with trailing non-name char (`$a-b`, `${a}b`) | rescanned as one generic expandable token | `$a` then `-b` | `ScanVariable@L2872-L3180` | +| quote inside a `-param` | rescan as generic | stop | `ScanParameter@L3185-L3338` | + +Stop-parsing (`--%`): the parser asks `GetVerbatimCommandArgument`, which reads raw text to end of line, stopping early at `|` or `&&` that is outside double quotes (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.GetVerbatimCommandArgument@L2283-L2315`). + +### 6.4 `-` and numbers + +`-`, `+`: `--` → `MinusMinus`, `-=` → `MinusEquals`; a following letter, `_` or `?` → `ScanParameter` (which may yield a dash operator in Expression mode); a digit or `.` → `ScanNumber` **only if `AllowSignedNumbers`** is set; otherwise `Minus` (`FACT-SRC`, `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4745-L4772`). `AllowSignedNumbers` is set to true by the parser's `UnaryExpressionRule` for the duration of that rule and restored afterwards, and once by a static helper that parses a number string (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.UnaryExpressionRule@L7029-L7226` (L7059-L7088), `src/System.Management.Automation/engine/parser/Parser.cs::Parser@L244-L252`). `INFERENCE`: `1 -1` therefore tokenizes `-1` as a number only where the grammar is at a unary position; in a binary position the parser did not set the flag for that token, so it is `Minus` + `1`. + +### 6.5 Statement and expression termination + +The tokenizer returns `NewLine` and `Semi` tokens and does not decide what they mean. It offers helpers: `SkipNewlines(skipSemis)` consumes whitespace, newlines, optionally `;`, comments and backtick continuations (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.SkipNewlines@L898-L974`); `IsPipeContinuation` / `ContinuationAfterExtent` look past blank text for a following `|` and return false when a blank or whitespace-only line intervenes ("blank or whitespace-only lines are not allowed in automatic line continuation") (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ContinuationAfterExtent@L1367-L1431`). The rule that a newline ends a statement is therefore a parser rule; see [02.2-parser.md](02.2-parser.md). + +### 6.6 Pushback and rescan + +`GetRestorePoint()` returns the current token start; `Resync(int)` recomputes `_currentIndex` (accounting for `_skippedCharOffsets`), clears `FirstToken` if it starts at or after the new position, and removes saved tokens ending after it from `TokenList` and `RequiresTokens`. `Resync` does **not** reset `LastToken` or `Mode` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Resync@L1039-L1078`, `RemoveTokensFromListDuringResync@L1080-L1100`). This is the whole backtracking mechanism: the parser re-asks for the same characters in another mode. + +## 7. Algorithms + +### 7.1 Numeric literals (`ScanNumberHelper`, `TryGetNumberValue`) + +**Numeric literal rules.** A number is an optional sign (`-`, en/em dash/horizontal bar, or `+`; only when `AllowSignedNumbers`), then either `.digits[exponent]`, `0x` hex digits, `0b` binary digits, or decimal digits with optional `.digits` and `e/E[+|-]digits`; then an optional type suffix (`u s l d y n`, plus a second character only after `u`: `ul us uy`), then an optional multiplier `k m g t p` that **must be followed by `b`/`B`** (powers of 1024; the suffix comes before the multiplier); the literal must be followed by a token terminator (in Expression mode also by a binary-operator start character) or the text is not a number and is rescanned as a generic token. Value typing: no suffix → `int`, else `long`, else (decimal format only) `decimal`, else `double`; real literals → `double` unless suffix `d` (`decimal`); hex and binary are two's complement with the sign decided by the top digit (`0xFFFFFFFF` → `-1`, `0xFF` → `255`), a 16-hex-digit or 64-bit-digit literal without `u` becomes `long`; the multiplier is applied to the value before the range cast and failure to fit returns `false` (reported as `BadNumericConstant` in Expression mode, otherwise the text becomes a generic token). + +Details (`FACT-SRC`, `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163`, `Tokenizer.TryGetNumberValue@L3555-L3882`, `Tokenizer.ScanNumber@L3884-L3931`): + +1. Leading `.` → `ScanNumberAfterDot`; a decimal number followed by `..` ungets the first dot so `1..2` is `1`, `..`, `2` (range operator). +2. `0x`: the scanner prepends `0` to the digit string so the top bit acts as sign; `0x` alone has no digits to validate (the emptiness test cannot fire) — behavior of `0x` alone is `UNVERIFIED`. +3. Suffix table: `y` sbyte, `uy` byte, `s` short, `us` ushort, `l` long, `ul` ulong, `u` uint (else ulong), `d` decimal, `n` BigInteger (`NumberSuffixFlags`, [TOKEN-CATALOG](TOKEN-CATALOG.md) §8). Signed non-hex values are range-checked after multiplication. +4. Multiplier factors: `k`=1024, `m`=1024², `g`=1024³, `t`=1024⁴, `p`=1024⁵. +5. Real literal with an integer suffix: the `double` is rounded (`Utils.AsBigInt`) and cast; failure → not a number (`src/System.Management.Automation/engine/Utils.cs::Utils@L42-L162`, notes). +6. Dash characters in sign/exponent position are normalized to `-`. + +### 7.2 Strings + +| Kind | Rules | Evidence | +|---|---|---| +| single-quoted | terminated by any single-quote-class char; a doubled quote yields one quote; backtick is **not** an escape; unterminated → `TerminatorExpectedAtEndOfString` | `ScanStringLiteral@L2317-L2360` | +| double-quoted (expandable) | doubled quote = one quote; `` ` `` escapes via `Backtick`; `$name`, `${..}`, `$(..)` become nested tokens; `{`/`}` are doubled in the stored `FormatString` and each nested part is replaced by `{N}` | `ScanStringExpandable@L2449-L2516`, `ScanDollarInStringExpandable@L2518-L2566` | +| here-string `@'..'@` / `@".."@` | header `@'` / `@"` must be followed (after optional whitespace) by a newline; the footer is quote + `@` **at the start of a line**; content is everything between, with the last newline (LF or CRLF) removed; expandable variant supports `$` and backtick but a lone `"` is literal | `ScanAfterHereStringHeader@L2578-L2635`, `ScanHereStringLiteral@L2685-L2753`, `ScanHereStringExpandable@L2755-L2865` | +| sub-expression in a string | `$(` ... `)` is pre-scanned by **counting parentheses only** (quotes and comments are not understood at this stage) and re-tokenized later by a nested scan; doubled `""` and `` `" `` inside it are collapsed with their offsets recorded so extents stay correct | `ScanSubExpression@L2362-L2447`, `StartNestedScan@L799-L822` | +| escapes (`Backtick`) | `` `0 `` NUL, `` `a `` BEL, `` `b ``, `` `e `` ESC, `` `f ``, `` `n ``, `` `r ``, `` `t ``, `` `v ``, `` `u{HHHHHH} `` (1-6 hex digits, up to U+10FFFF, surrogate pair emitted for > U+FFFF), any other char = itself | `Backtick@L1463-L1492`, `ScanUnicodeEscape@L1494-L1588` | + +The `INFERENCE` that follows from paren-only counting: a `)` inside a quoted string within `$(...)` inside a string ends the pre-scan early. No test was read for this; `UNVERIFIED` (backlog). + +### 7.3 Variables (`ScanVariable`) + +(`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanVariable@L2872-L3180`) + +- `${ ... }`: reads to `}`; backtick escapes allowed; an unescaped `{` inside is an error but kept; empty name is an error (`EmptyVariableReference`); EOF → `IncompleteDollarVariableReference`. In Command mode, if the char after `}` is not a terminator (and not `.` or `[`), the text is rescanned as a generic token. +- `$name`: name chars are letters, digits, `_`, `?`, and `:` (a `:` followed by another `:` stops the name — `$a::b`); `$$`, `$?`, `$^` are single-char names. `-` is **not** a name char, so `$a-b` is `$a` then `-b` in Expression mode but one generic token in Command mode. +- `@name` produces a splatted variable (`SplattedVariable`). `$using:`/scope prefixes are not parsed here: the full text goes into `VariablePath`, which interprets drive/scope qualifiers (`INFERENCE`; `VariablePath` is outside this file). +- Empty unqualified path → `InvalidVariableReference` (or the `...WithDrive` message for `$env:`). + +### 7.4 Comments and `#requires` + +- Line comment: `#` to end of line, token kind `Comment`, never returned by `NextToken` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanLineComment@L1792-L1807`). +- Block comment: `<# ... #>`, not nested; EOF → `MissingTerminatorMultiLineComment` (`ScanBlockComment@L1809-L1837`). +- `#requires`: while scanning a line comment the tokenizer runs a small state machine that matches the letters `requires` **immediately** after `#`; any other char (including whitespace) breaks the match; a match records the comment token in `RequiresTokens` (only for the main script, not nested scans) (`ScanToEndOfCommentLine@L1605-L1754`, `ScanLineComment@L1792-L1807`). The recognition does not check the character after `requires` (`FACT-SRC` code shape; behavioral consequence `INFERENCE`). +- Processing of `#requires` lines happens in `GetScriptRequirements` by re-running the command parser on the comment text (`@L1894-L1987`); parameters are matched by **prefix** on the text (`StartsWith`) in the order runasadministrator, shellid, pssnapin, psedition, version, assembly, modules; arguments must be constants (`HandleRequiresParameter@L1997-L2214`). Candidate to omit for MicroteamShell (§18, LSN-TOK-07). +- Signature blocks: a comment line whose whitespace-stripped lowercase text is within Levenshtein distance < 10 (bounds: length 14..34) of `sig#beginsignatureblock\n` sets `_beginSignatureExtent`; later code triggers `TokenAfterEndOfValidScriptText` (`CheckAstIsBeforeSignature@L1133-L1144`, `ScanToEndOfCommentLine@L1605-L1754`). Security-relevant (§14). + +### 7.5 Redirection, generic tokens, identifiers, labels + +- `NewFileRedirectionToken(from, append, fromSpecifiedExplicitly)`: in **Expression** mode an explicit stream number (`2>`) is handled by ungetting the `>` and returning a `Number` token — so `2>x` in Expression mode is number `2` followed by `>` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NewFileRedirectionToken@L1300-L1315`). +- Generic token: continues until a `ForceStartNewToken` char (whitespace, NUL, `;`, `&`, `(`, `)`, `,`, `{`, `|`, `}`); `#`, `@`, `<`, `>`, `[`, `]`, `=` do **not** end it; quotes and `$` inside are scanned inline, so a generic token may carry nested tokens (an expandable generic) (`ScanGenericToken@L3382-L3479`). Dynamic keywords are recognized at the end of a generic scan. +- Identifier: `ScanIdentifier` reads `IsIdentifierFollow` chars; TypeName mode → `ScanTypeName` (adds `. _ + # \ \``) ; Command mode with a non-terminator next → generic token; keywords are looked up only when `!WantSimpleName` and mode is Command or Signature; `inlinescript` only when `InWorkflowContext` (`ScanIdentifier@L4402-L4458`). +- Member access (parser-driven, no whitespace skipping): `GetMemberAccessOperator(allowLBracket)` yields `Dot`, `ColonColon`, `QuestionDot`, `QuestionLBracket`, `LBracket`; block comments are skipped "for backward compatibility"; `GetInvokeMemberOpenParen` accepts `(` or `{` immediately after a member name (`@L4167-L4262`, `@L4264-L4283`). +- Attribute/type `[`: `GetLBracket` exists because in Command mode `[` can start a generic token (`@L4285-L4365`). +- Assembly-qualified type names are tokenized by `GetAssemblyNameSpec` (`@L4519-L4572`). + +### 7.6 Position tracking + +`Initialize` builds a line-start map; **only `\r`, `\n` and `\r\n`** end lines (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797`). `CurrentExtent()` computes start/end from `_tokenStart`/`_currentIndex` plus `_nestedTokensAdjustment`, then corrects for characters removed from nested sub-expression text: each set bit in `_skippedCharOffsets` before the start shifts both ends by one, bits inside extend the end (`CurrentExtent@L1181-L1207`). `GetChar` increments the index even past the end and returns `\0`; `AtEof` is therefore `_currentIndex > _script.Length` (`GetChar@L837-L881`). The `EndOfInput` token's extent is one past the end of the script (`NextToken`, `\0` case). + +## 8. Observable behavior [SPEC] + +No golden sample: STATIC-ONLY. The behaviors in §6-§7 are read from source; where repo tests assert them, the tests are in `test/powershell/Language/Parser/` (inventory in `data/tests.json`; a per-assertion mapping is `UNVERIFIED` and listed in §19). Examples whose shape follows directly from the dispatcher: + +- `echo a-b` → one generic argument `a-b`; `1+2` → `1`, `+`, `2`; `echo -eq` → a `Parameter` token named `eq` (Command mode). +- `$a-b` in an expression → variable `$a` then `-b` (a `Parameter` token, or a dash operator if the text matches); in a command → one generic token. +- `1kb` → number 1024; `1k` → not a number (generic). +- `` "a`n$x" `` → expandable string with nested variable `x`. + +All labelled `INFERENCE` until verified with `pwsh` (corpus candidates planned for the verification phase). + +## 9. Error handling and edge cases + +- Errors are reported through the parser (`ReportError`, `ReportIncompleteInput`), not thrown: the tokenizer continues and marks the token `TokenInError` (`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ReportError@L1146-L1164`, `ReportIncompleteInput@L1166-L1174`). Examples: unterminated strings, unterminated block comment, bad Unicode escape (`InvalidUnicodeEscapeSequence` family), too-many hex digits, unexpected characters after a here-string header (recovery: skip to a same-quote-plus-`@` terminator or end of line). +- `BadNumericConstant` is reported only in Expression mode; in other modes an invalid number becomes a generic word. +- Deep recursion in `ScanSubExpression` is guarded by `RuntimeHelpers.EnsureSufficientExecutionStack()` (`@L2362-L2447`). +- `AtEof` semantics and `\0` handling (§7.6): a literal NUL char inside the script is a generic token. +- Pitfall candidates (code shape): `DynamicKeywordExtension.IsMetaDSCResource` discards a computed value (`src/System.Management.Automation/engine/parser/tokenizer.cs::DynamicKeywordExtension@L359-L379`) — not tokenizing; `0x` alone (§7.1); `#requires` suffix (§7.4). + +## 10. Concurrency and threading + +The dynamic-keyword table is `[ThreadStatic]` (`src/System.Management.Automation/engine/parser/tokenizer.cs::DynamicKeyword@L71-L127`), so keyword recognition depends on the thread that parses. A `Tokenizer` instance is owned by one `Parser` and is not thread-safe; the string-builder pool is a static `Queue` capped at 10 buffers (capacity < 1024) (`@L1758-L1788`) — `INFERENCE`: pooled instances are accessed from the owning parse only; thread safety of the pool itself was not analysed (`UNVERIFIED`). + +## 11. Extension points and customization hooks + +Add a keyword: add rows to `s_keywordText`/`s_keywordTokenKind`, a `TokenKind` member, and rows to `s_staticTokenFlags`/`s_tokenText` (the `DEBUG` assertions will fail otherwise). Add a dash operator: rows in `_operatorText`/`s_operatorTokenKind` plus the kind tables and precedence flag. Detailed recipes are in HOWTO-extend-language (planned, T210). Dynamic keywords (DSC) add keywords at run time. + +## 12. Dependencies + +Inbound: `Parser` (all calls), `ScriptBlock`/completion APIs via `Parser.ParseInput`/`Tokenize`. Outbound: `PositionHelper`, `InternalScriptExtent`, `VariablePath`, `Utils.AsBigInt/TryCast/ParseBinary`, `LanguagePrimitives.ConvertTo` (`#requires`), `DynamicKeyword`/DSC subsystem (`ICrossPlatformDsc`). See the `using` list at `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L4-L17`. + +## 13. Performance notes + +`StringBuilder` pooling in the scanner (`GetStringBuilder`/`Release`, `@L1763-L1790`); the char-class table is a flat 128-entry array; the `Resync` design rescans text instead of keeping parallel token streams (`INFERENCE`: cheap because lookahead is one token). + +## 14. Security considerations (mandatory) + +- The tokenizer enforces no policy. It **does** carry security-relevant behavior: signature-block lookalike detection (Levenshtein threshold 10 on text between 14 and 34 characters) exists "to disallow code after anything that looks like a signature" so trailing code cannot hide behind a fake signature (comments `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanToEndOfCommentLine@L1605-L1754`; wording `FACT-SRC` for the code, intent from comments). +- Unicode normalization surface: smart quotes and en/em dash are treated as ASCII quotes/dashes (§5), which is a visible-vs-actual-text risk for copy-pasted scripts (`INFERENCE`). +- Static `TokenFlags.DisallowedInRestrictedMode` on kinds feeds the constrained/restricted-language checks done elsewhere ([TOKEN-CATALOG](TOKEN-CATALOG.md) §3-4). +- `#requires` evaluation is performed at parse time (tokenizer-owned) and can name modules/assemblies but does not load them here (`UNVERIFIED`: loading occurs later). + +## 15. Enterprise considerations + +Observability: tokens can be saved in `TokenList` with comments for tooling; testability: pure function of text + mode, so the whole tokenizer can be unit-tested if mode sequencing is simulated; resource limits: recursion guard only; no limit on token length. Missing: no token-length or nesting-depth limit beyond the stack check. + +## 16. Tests + +Parser/language tests: `test/powershell/Language/Parser/` (counts in `data/tests.json`, `APPROX`). Tokenizer-focused: token tests and number/string tests in that folder (exact assertion mapping `UNVERIFIED`). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| on-demand, mode-driven tokenization with one-token lookahead and rescan | `FACT-SRC` (§6.2) | context-sensitive barewords without a separate command grammar; cost: the parser owns mode correctness and tokens cannot be tokenized in isolation | +| table-driven char classes for ASCII, `char.Is*` fallback above 127 | `FACT-SRC` | fast; non-ASCII letters/digits/dashes/quotes accepted in many positions | +| `$(...)` inside strings pre-scanned by paren counting, tokenized later | `FACT-SRC` | simple; mis-handles `)` in nested quotes (`UNVERIFIED`) | +| comments swallowed by `NextToken`, kept only in `TokenList` | `FACT-SRC` | parser never sees comments; tooling can | +| `#requires` and signature detection live in the tokenizer | `FACT-SRC` | tokenizer has side effects beyond tokens | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-TOK-01** [REPLICATE] (area: NLD-LEX) Keep a parser-selected `Mode` (Command / Expression / TypeName / Signature) on a pull-style tokenizer; initial mode Command; each switch restored in `finally`. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`. +- **LSN-TOK-02** [REPLICATE] (area: NLD-LEX) Treat `-name` as a parameter in Command mode and as dash-operator-or-parameter in Expression mode; recognize keywords only in Command/Signature mode. Evidence: `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanParameter@L3185-L3338`, `Tokenizer.ScanIdentifier@L4402-L4458`. +- **LSN-TOK-03** [REPLICATE] (area: NLD-LEX) Reproduce the generic-token rule in Command mode: operator characters and non-terminators glue into one word, with the exact terminator set (`ForceStartNewToken`). Evidence: `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanGenericToken@L3382-L3479`, `src/System.Management.Automation/engine/parser/CharTraits.cs::CharExtensions@L230-L413`. +- **LSN-TOK-04** [ADAPT] (area: NLD-LEX) Implement rescan with an explicit snapshot (position + mode) instead of resyncing from a token, but keep the one-token-lookahead contract so a mode switch re-reads the peeked token. Why: same observable behavior, simpler to reason about. Evidence: `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Resync@L1039-L1078`. +- **LSN-TOK-05** [REPLICATE] (area: NLD-LEX) Number literal grammar: suffix before multiplier, multiplier needs `b`, two's-complement hex/binary, int→long→decimal→double promotion. Evidence: `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882`. +- **LSN-TOK-06** [ADAPT] (area: NLD-LEX) Pre-scan `$(...)` inside strings with a tokenizer-aware scan (quotes and comments), not paren counting alone. Why: avoids the early-`)` edge case; observable behavior for valid scripts is identical. Evidence: `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanSubExpression@L2362-L2447`. +- **LSN-TOK-07** [AVOID] (area: NLD-LEX) Leave `#requires` processing and signature lookalike detection out of the tokenizer (move to a later phase or omit); replace with a dedicated pragma pass if the owner wants it. Evidence: `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.GetScriptRequirements@L1894-L1987`. DEV: DEV-LEX-001 + +## 19. Open questions / UNVERIFIED + +- `0x` alone as a number, `#requiresX` acceptance, early-`)` in nested sub-expression strings (all `UNVERIFIED`; corpus candidates to confirm with `pwsh`). +- Exact tests asserting tokenizer behavior (mapping from `test/powershell/Language/Parser/*.ps1` assertions). +- `src/System.Management.Automation/engine/parser/Parser.cs` call-site list in §6.2 is from the mode-switch grep (`FACT-SRC`) but the `why` column is `INFERENCE`. +- Token → parser-rule consumer mapping: answered per method by the generated production table in [02.2-parser.md](02.2-parser.md) §7.1 (tokens referenced, `APPROX`); a per-token inverse index is not produced. + +Verification: every method cited in §6-§7 was read in a ≤300-line window (notes `.work/notes/src__System.Management.Automation__engine__parser__tokenizer.cs.md`, W1-W16; `token.cs` W1-W5; `CharTraits.cs` whole file); mode-switch table re-derived by `grep -n "SetTokenizerMode(TokenizerMode\\."` on `Parser.cs` and mapped to methods with `data/parser-methods.json`; `NextToken` dash/plus cases re-read at L4724-L4772; `TokenKind` count reconciled with `data/numbers.json` (`TOKENKIND_MEMBERS`); catalog tables are `GENERATED` by `tools/gen_T21.py` with length assertions. diff --git a/docs-out/02-language/02.2-parser.md b/docs-out/02-language/02.2-parser.md new file mode 100644 index 0000000..46c498f --- /dev/null +++ b/docs-out/02-language/02.2-parser.md @@ -0,0 +1,435 @@ +--- +id: LANG-PARSER-001 +title: Parser +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Parser.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/parser-methods.json, data/resources.json, data/tables.json, data/enums.json] +--- + +# Parser + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Grammar: [GRAMMAR.ebnf.md](GRAMMAR.ebnf.md). Errors: [ERROR-CATALOG.md](ERROR-CATALOG.md). Tokens: [02.1-tokenizer.md](02.1-tokenizer.md), [TOKEN-CATALOG.md](TOKEN-CATALOG.md). + +## 1. Purpose and scope [SPEC] + +The parser turns script text into a `ScriptBlockAst`. It is a hand-written **recursive-descent parser** with a **one-token lookahead** that pulls tokens from the tokenizer on demand and tells the tokenizer which mode to use (see CF-LEX-001 in [02.1-tokenizer.md](02.1-tokenizer.md) §6.2). Binary expressions use precedence climbing with an operand stack and an operator stack, not one method per precedence level. It reports errors into a list and keeps going, so a script with errors still yields a (partial) AST. `FACT-SRC` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8191`). + +In scope: entry points, the rule structure, statement and expression parsing, command parsing, operator precedence, error recovery, parse-time side effects. Out of scope: AST classes ([02.3-ast.md](02.3-ast.md), planned T23), semantic checks run after the parse ([02.4-semantic-analysis.md](02.4-semantic-analysis.md), planned T24). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| `src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8191` | `public sealed class` | whole parser: 157 methods in `data/parser-methods.json`; every `*Rule` method is one production | A | +| `src/System.Management.Automation/engine/parser/Parser.cs::ParseError@L8193-L8241` | `public class` | error record: `Extent`, `ErrorId`, `Message`, `IncompleteInput` | A | +| `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource@L8247-L8278` | event source | ETW events ParseStart/Stop, ResolveSymbols, SemanticChecks, CheckSecurity, Compile | A | +| `src/System.Management.Automation/engine/parser/Parser.cs::ParseMode@L26-L30` | enum | `Default`, `ModuleAnalysis` (the latter skips post-parse semantic checks) | A | +| `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer@L594-L5083` | class | token source, see [02.1-tokenizer.md](02.1-tokenizer.md) | A | + +## 3. Responsibilities and non-responsibilities + +Responsible for: choosing tokenizer modes, building the AST, error reporting and recovery, parse-time handling of `using`, `class`, `enum`, `configuration` and dynamic keywords, running `PerformPostParseChecks` after the parse. Not responsible for: tokenizing, semantic checks (called, not implemented here), execution. + +## 4. Key types and contracts [IMPL] + +**Public entry points** (`FACT-SRC`): + +| API | Behavior | Evidence | +|---|---|---| +| `Parser.ParseFile(fileName, out tokens, out errors)` | reads the file (`ExternalScriptInfo.ScriptContents`); on read failure returns an empty `ScriptBlockAst` and a `FileReadError` error; special handling for `*.schema.psm1` DSC files | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseFile@L66-L120` | +| `Parser.ParseInput(input, out tokens, out errors)` (+ overload with file name) | parses a string; `null` input → `ArgumentNullException` | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseInput@L131-L162` | +| internal `Parse` → `ParseTask` | `tokenizer.Initialize`, `ErrorList.Clear()`, `ScriptBlockRule(null, isFilter: false)`, `GetScriptRequirements()`, then `PerformPostParseChecks` when `ParseMode.Default` | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227` | +| static helpers `ScanNumber`, `ScanType`, `ScanString` | parse a number/type/string from text for runtime use (e.g. `LanguagePrimitives`); throw `ParseException` if errors | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScanNumber@L230-L268`, `Parser.ScanType@L270-L290`, `Parser.ScanString@L292-L305` | + +Any exception that is not a syntax error escaping `ParseFile`/`ParseInput` becomes `ParseException(UnrecoverableParserError)` (notes W1; see CF-ERR-002 in [ERROR-CATALOG.md](ERROR-CATALOG.md)). + +**State** (`src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L60`): `_tokenizer`, `_ungotToken` (the single pushed-back token), `_disableCommaOperator`, `_savingTokens`, `_inConfiguration`, `_parseMode`, `_fileName`, `ErrorList`. Constants `VERBATIM_ARGUMENT = "--%"` and `VERBATIM_PARAMETERNAME = "-%"`. + +**Token plumbing**: `NextToken`/`PeekToken`/`UngetToken` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.NextToken@L434-L446`; unget asserts only one token of lookahead, `@L493-L497`); `SetTokenizerMode` (rescans the peeked token when the mode changes, §6.3); `NextMemberAccessToken`, `NextInvokeMemberToken`, `NextLBracket`, `GetVerbatimCommandArgumentToken` return `null` when a token is already pushed back (error state) (`@L448-L485`); `SkipNewlines` only calls the tokenizer if no token is pending or the pending token is a `NewLine` (`@L348-L355`). + +## 5. Data model + +- The parser produces AST nodes (catalog in [AST-CATALOG.md](AST-CATALOG.md), planned T23): `ScriptBlockAst` → `ParamBlockAst`, `NamedBlockAst`, `StatementBlockAst` → statements (`PipelineAst`, `PipelineChainAst`, `AssignmentStatementAst`, `IfStatementAst`, loops, `TryStatementAst`, `FunctionDefinitionAst`, `TypeDefinitionAst`, `UsingStatementAst`, ...) → `CommandAst` / `CommandExpressionAst` → expressions. +- Error nodes: `ErrorStatementAst` and `ErrorExpressionAst` hold the sub-nodes parsed before the error so tools (completion) can still use them. +- Positions: every node gets an extent built by helpers `ExtentOf`, `Before`, `After`, `ExtentFromFirstOf`, `LastCharacterOf` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L572-L702`). +- `$^` and `$$` are updated from the first/last token of each statement via `SetPreviousFirstLastToken` (ignores tokens whose text is `$^` or `$$`) (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetPreviousFirstLastToken@L317-L342`). + +## 6. Control flow + +### 6.1 Overall structure + +```mermaid +flowchart TD + P[ParseInput / ParseFile] --> T[ParseTask] + T --> SB[ScriptBlockRule] + SB --> U[UsingStatementsRule] + SB --> PB[ParamBlockRule] + SB --> BD[ScriptBlockBodyRule] + BD --> NB[NamedBlockListRule] + BD --> SL[StatementListRule] + SL --> ST[StatementRule] + ST --> KW[keyword statements: if while for foreach switch try trap function class enum using data] + ST --> PC[PipelineChainRule] + PC --> EX[ExpressionRule] + PC --> PL[PipelineRule] + PL --> CM[CommandRule] + EX --> BE[BinaryExpressionRule] + BE --> AL[ArrayLiteralRule] + AL --> UN[UnaryExpressionRule] + UN --> PR[PrimaryExpressionRule] +``` + +`ScriptBlockRule(lCurly, isFilter)`: using statements only for the top-level script (no `{`), then a **speculative** `ParamBlockRule` (restore point + `Resync` if no `param (` follows, so a command named `param` is not an error), then the body (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScriptBlockRule@L777-L809`). A body with a block-name keyword (`begin process end dynamicparam clean`, flagged `ScriptBlockBlockName`) goes to `NamedBlockListRule`, otherwise to a loop of `StatementListRule`/`CompleteScriptBlockBody` (`@L1682-L1725`, `@L1727-L1857`). At top level, a stray token that cannot start a statement produces `UnexpectedToken`, is **eaten**, and parsing loops (`CompleteScriptBlockBody`, `@L1629-L1680`). + +### 6.2 Commands vs expressions at the start of a statement + +`StatementRule` switches on the first token kind (keyword → keyword rule; label; `class`/`enum`/`using`/dynamic keyword) and otherwise calls `PipelineChainRule` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementRule@L1950-L2159`). `PipelineChainRule` **first** switches to Expression mode and tries `ExpressionRule`; the outcome decides everything: + +1. Expression found and next token has the `AssignmentOperator` trait → `AssignmentStatementAst`, right side is `StatementRule()` after newlines (so `$x = if (...) {...}` and `$x = 1 | f` work) (`@L5786-L5840`). +2. Expression found, no assignment → it becomes the first element of the pipeline (`PipelineRule(startExpression)`). +3. No expression (e.g. a bareword such as `Get-Item`) → `PipelineRule()` parses a `CommandRule` in Command mode. + +This is the parser-side answer to command-vs-expression ambiguity: **try expression first in Expression mode; fall back to command**. The tokenizer mode switch discards and rescans the pending lookahead token because identifier-like text tokenizes differently in the two modes (CF-LEX-001). `INFERENCE`: because expression parsing runs first, anything that parses as an expression (a number, string, variable, `(...)`, `[type]`...) is never a command name unless it is introduced by `&` or `.`. + +### 6.3 Pipelines, chains and terminators + +- `PipelineChainRule` loops: parse a pipeline; if the next token is `&&` or `||`, skip newlines after it, build a left-nested `PipelineChainAst`; `&` ends the chain and sets `Background` (an `&` followed by `&&`/`||` is `BackgroundOperatorInPipelineChain`); with no operator it returns the plain `PipelineAst` (no wrapper node) (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988`). +- `PipelineRule` parses elements separated by `|`; after `|` newlines are skipped; a **newline followed by a line starting with `|`** also continues the pipeline (`IsPipeContinuation`, [02.1-tokenizer.md](02.1-tokenizer.md) §6.5). An element that starts with an expression after the first element is `ExpressionsMustBeFirstInPipeline` (a semantic error, parse continues). A pipeline stops without error at `;`, newline, `)`, `}`, `&&`, `||`, end of input; any other token → `UnexpectedToken` (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineRule@L5990-L6160`). +- **Statement and terminator rule** (canonical fact CF-GRM-001, owner [GRAMMAR.ebnf.md](GRAMMAR.ebnf.md) §3a): statements are separated by optional runs of newlines and semicolons; no terminator is required after a statement that ends in `}`; a pipeline ends at newline, `;`, `)`, `}`, `&&`/`||`, `&` or end of input; newlines continue a construct only after the places listed in the fact. +- `StatementListRule` collects `TrapStatementAst` into a separate list from ordinary statements (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944`). + +### 6.4 Tokenizer-mode switching points + +The parser selects Command mode for `CommandRule`, single command arguments and the bodies of `{ }`, `$( )`, `( )`; Expression mode for `ExpressionRule`, `BinaryExpressionRule`, parameter defaults, attribute arguments and hashtable values; TypeName mode for `[...]`; Signature mode for class/enum/method headers. The full table with line numbers is in [02.1-tokenizer.md](02.1-tokenizer.md) §6.2. Each switch is restored in a `finally` block. + +## 7. Algorithms + +### 7.1 Production table (every method) + + +| Production (method) | Kind | File@Lines | Tokens referenced (`TokenKind`, APPROX) | AST created (`new …Ast(`, APPROX) | Error IDs (`ParserStrings`, APPROX) | +|---|---|---|---|---|---| +| `Parser` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.Parser@L52-L57` | — | — | — | +| `ParseFile` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseFile@L66-L120` | — | `ScriptBlockAst`, `StatementBlockAst` | — | +| `ParseInput` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseInput@L131-L134` | — | — | — | +| `ParseInput` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseInput@L144-L162` | — | — | — | +| `Parse` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.Parse@L164-L174` | — | — | — | +| `ParseTask` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227` | — | — | `ScriptTooComplicated` | +| `ScanNumber` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScanNumber@L230-L268` | — | — | — | +| `ScanType` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScanType@L270-L290` | — | — | — | +| `ScanString` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScanString@L292-L305` | — | — | — | +| `IgnoreTokenWhenUpdatingPreviousFirstLast` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.IgnoreTokenWhenUpdatingPreviousFirstLast@L310-L315` | `Generic`, `Variable` | — | — | +| `SetPreviousFirstLastToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetPreviousFirstLastToken@L317-L342` | — | — | — | +| `SkipNewlines` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SkipNewlines@L348-L355` | `NewLine` | — | — | +| `SkipNewlinesAndSemicolons` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SkipNewlinesAndSemicolons@L357-L364` | `NewLine`, `Semi` | — | — | +| `SyncOnError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SyncOnError@L366-L432` | `EndOfInput`, `LBracket`, `LCurly`, `LParen`, `RBracket`, `RCurly`, `RParen` | — | — | +| `NextToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.NextToken@L434-L439` | — | — | — | +| `PeekToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.PeekToken@L441-L446` | — | — | — | +| `NextMemberAccessToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.NextMemberAccessToken@L448-L454` | — | — | — | +| `NextInvokeMemberToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.NextInvokeMemberToken@L456-L462` | — | — | — | +| `NextLBracket` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.NextLBracket@L464-L474` | `LBracket` | — | — | +| `GetVerbatimCommandArgumentToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetVerbatimCommandArgumentToken@L476-L485` | `Parameter` | — | — | +| `SkipToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SkipToken@L487-L491` | — | — | — | +| `UngetToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.UngetToken@L493-L497` | — | — | — | +| `SetTokenizerMode` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L536` | `EndOfInput` | — | — | +| `Resync` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.Resync@L538-L542` | — | — | — | +| `Resync` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.Resync@L544-L548` | — | — | — | +| `IsSpecificParameter` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.IsSpecificParameter@L550-L555` | `Parameter` | — | — | +| `RequireStatementTerminator` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.RequireStatementTerminator@L557-L570` | `EndOfInput`, `NewLine`, `Semi` | — | `MissingStatementTerminator` | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L572-L583` | — | — | — | +| `Before` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.Before@L585-L592` | — | — | — | +| `After` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.After@L594-L600` | — | — | — | +| `LastCharacterOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.LastCharacterOf@L602-L613` | — | — | — | +| `ExtentFromFirstOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentFromFirstOf@L615-L646` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L648-L648` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L650-L650` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L652-L652` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L654-L654` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L656-L656` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L658-L658` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L660-L660` | — | — | — | +| `ExtentOf` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExtentOf@L662-L662` | — | — | — | +| `Before` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.Before@L665-L665` | — | — | — | +| `After` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.After@L667-L667` | — | — | — | +| `After` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.After@L669-L669` | — | — | — | +| `GetNestedErrorAsts` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetNestedErrorAsts@L671-L702` | — | — | — | +| `TryParseAsConstantHashtable` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.TryParseAsConstantHashtable@L710-L766` | — | — | — | +| `ScriptBlockRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScriptBlockRule@L772-L775` | — | — | — | +| `ScriptBlockRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScriptBlockRule@L777-L809` | — | — | — | +| `UsingStatementsRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.UsingStatementsRule@L811-L843` | `Using` | — | — | +| `ParamBlockRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParamBlockRule@L845-L912` | `LParen`, `Param`, `RParen` | `ParamBlockAst` | `MissingEndParenthesisInFunctionParameterList`, `TypeNotAllowedBeforeParam` | +| `ParameterListRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParameterListRule@L914-L952` | `Comma` | — | `MissingExpressionAfterToken` | +| `ParameterRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParameterRule@L954-L1025` | `Equals`, `RParen`, `SplattedVariable`, `Variable` | `ParameterAst`, `VariableExpressionAst` | `InvalidFunctionParameter`, `MissingExpressionAfterToken` | +| `AttributeListRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.AttributeListRule@L1027-L1052` | — | — | — | +| `AttributeRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.AttributeRule@L1054-L1153` | `Identifier`, `LParen`, `RBracket`, `RParen` | `AttributeAst`, `TypeConstraintAst` | `EndSquareBracketExpectedAtEndOfAttribute`, `MissingEndParenthesisInExpression`, `MissingTypename` | +| `AttributeArgumentsRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.AttributeArgumentsRule@L1155-L1269` | `Comma`, `Equals`, `NewLine`, `RBracket`, `RParen` | `ConstantExpressionAst`, `ErrorExpressionAst`, `NamedAttributeArgumentAst` | `DuplicateNamedArgument`, `MissingExpressionAfterToken`, `MissingExpressionInNamedArgument` | +| `TypeNameRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.TypeNameRule@L1271-L1331` | `Identifier` | — | — | +| `FinishTypeNameRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.FinishTypeNameRule@L1333-L1395` | `Comma`, `EndOfInput`, `Identifier`, `LBracket`, `RBracket` | — | `MissingAssemblyNameSpecification`, `MissingTypename`, `UnexpectedToken` | +| `GetSingleGenericArgument` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetSingleGenericArgument@L1397-L1432` | `Identifier`, `LBracket`, `RBracket` | — | `EndSquareBracketExpectedAtEndOfType`, `MissingTypename` | +| `GenericTypeArgumentsRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GenericTypeArgumentsRule@L1434-L1473` | `Comma`, `Identifier`, `LBracket` | — | `MissingTypename` | +| `GenericTypeNameRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GenericTypeNameRule@L1475-L1521` | `Comma`, `LBracket`, `RBracket` | — | `EndSquareBracketExpectedAtEndOfAttribute`, `MissingAssemblyNameSpecification` | +| `CompleteArrayTypeName` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CompleteArrayTypeName@L1523-L1627` | `Comma`, `EndOfInput`, `LBracket`, `RBracket` | — | `ArrayHasTooManyDimensions`, `EndSquareBracketExpectedAtEndOfAttribute`, `MissingAssemblyNameSpecification`, `UnexpectedToken` | +| `CompleteScriptBlockBody` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CompleteScriptBlockBody@L1629-L1680` | `EndOfInput`, `RCurly` | — | `MissingEndCurlyBrace`, `UnexpectedToken` | +| `ScriptBlockBodyRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScriptBlockBodyRule@L1682-L1725` | — | `ScriptBlockAst`, `StatementBlockAst` | — | +| `NamedBlockListRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.NamedBlockListRule@L1727-L1857` | `Begin`, `Clean`, `Dynamicparam`, `End`, `EndOfInput`, `Process`, `RCurly` | `NamedBlockAst`, `ScriptBlockAst`, `StatementBlockAst` | `DuplicateScriptCommandClause`, `MissingNamedBlocks`, `MissingNamedStatementBlock` | +| `StatementBlockRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementBlockRule@L1859-L1895` | `LCurly`, `RCurly` | `StatementBlockAst` | `MissingEndCurlyBrace` | +| `StatementListRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944` | `RCurly`, `RParen` | — | — | +| `StatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementRule@L1950-L2159` | `Break`, `Catch`, `Class`, `Configuration`, `Continue`, `Data`, `Define`, `Do`, `DynamicKeyword`, `Else`, `ElseIf`, `EndOfInput`, `Enum`, `Exit` +20 | `ErrorStatementAst` | `ReservedKeywordNotAllowed`, `TypeNotAllowedBeforeStatement`, `UnexpectedAttribute`, `UsingMustBeAtStartOfScript` | +| `SimpleNameRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SimpleNameRule@L2161-L2165` | — | — | — | +| `SimpleNameRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SimpleNameRule@L2167-L2190` | `Identifier` | `StringConstantExpressionAst` | — | +| `LabelOrKeyRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.LabelOrKeyRule@L2192-L2234` | `NewLine`, `Semi` | — | — | +| `BreakStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BreakStatementRule@L2236-L2247` | — | `BreakStatementAst` | — | +| `ContinueStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ContinueStatementRule@L2249-L2260` | — | `ContinueStatementAst` | — | +| `ReturnStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReturnStatementRule@L2262-L2272` | — | `ReturnStatementAst` | — | +| `ExitStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExitStatementRule@L2274-L2284` | — | `ExitStatementAst` | — | +| `ThrowStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ThrowStatementRule@L2286-L2297` | — | `ThrowStatementAst` | — | +| `LabeledStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.LabeledStatementRule@L2299-L2339` | `Do`, `For`, `Foreach`, `Switch`, `While` | — | — | +| `BlockStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BlockStatementRule@L2341-L2359` | — | `BlockStatementAst`, `ErrorStatementAst` | `MissingStatementAfterKeyword` | +| `InlineScriptRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.InlineScriptRule@L2370-L2398` | `LCurly` | `StringConstantExpressionAst` | `MissingStatementAfterKeyword` | +| `IfStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.IfStatementRule@L2400-L2541` | `Else`, `ElseIf`, `LParen`, `RParen` | `ErrorStatementAst`, `IfStatementAst` | `IfStatementMissingCondition`, `MissingEndParenthesisAfterStatement`, `MissingOpenParenthesisInIfStatement`, `MissingStatementBlock`, `MissingStatementBlockAfterElse` | +| `SwitchStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SwitchStatementRule@L2543-L2891` | `Default`, `EndOfInput`, `LCurly`, `LParen`, `Minus`, `Parameter`, `RCurly`, `RParen` | `CommandExpressionAst`, `ErrorStatementAst`, `PipelineAst`, `StringConstantExpressionAst`, `SwitchStatementAst` | `InvalidSwitchFlag`, `MissingCurlyBraceInSwitchStatement`, `MissingEndCurlyBrace`, `MissingEndParenthesisInSwitchStatement`, `MissingFilenameOption`, `MissingSwitchConditionExpression`, `MissingSwitchStatementClause`, `MultipleSwitchDefaultClauses` +1 | +| `ConfigurationStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ConfigurationStatementRule@L2893-L3322` | `Comma`, `EndOfInput`, `LCurly` | `ConfigurationDefinitionAst`, `ErrorStatementAst` | `ConfigurationBodyEmpty`, `ConfigurationNotAllowedInConstrainedLanguage`, `ConfigurationNotAllowedOnArm64`, `ConfigurationNotAllowedOnWinPE`, `InvalidConfigurationName`, `MissingConfigurationName`, `MissingCurlyInConfigurationStatement`, `ParserError` | +| `GetWordOrExpression` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetWordOrExpression@L3331-L3356` | `EndOfInput` | — | `ParameterRequiresArgument`, `RequiredNameOrExpressionMissing` | +| `ForeachStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ForeachStatementRule@L3358-L3518` | `In`, `LParen`, `Parameter`, `RParen`, `SplattedVariable`, `Variable` | `ErrorStatementAst`, `ForEachStatementAst`, `VariableExpressionAst` | `InvalidForeachFlag`, `MissingEndParenthesisAfterForeach`, `MissingForeachExpression`, `MissingForeachStatement`, `MissingInInForeach`, `MissingOpenParenthesisAfterKeyword`, `MissingThrottleLimit`, `MissingVariableNameAfterForeach` | +| `ForStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ForStatementRule@L3520-L3626` | `LParen`, `RParen`, `Semi` | `ErrorStatementAst`, `ForStatementAst` | `MissingEndParenthesisAfterStatement`, `MissingLoopStatement`, `MissingOpenParenthesisAfterKeyword` | +| `WhileStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.WhileStatementRule@L3628-L3708` | `LParen`, `RParen` | `ErrorStatementAst`, `WhileStatementAst` | `MissingEndParenthesisAfterStatement`, `MissingExpressionAfterKeyword`, `MissingLoopStatement`, `MissingOpenParenthesisAfterKeyword` | +| `DynamicKeywordStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.DynamicKeywordStatementRule@L3724-L4090` | `Dot`, `DynamicKeyword`, `EndOfInput`, `Identifier`, `LCurly` | `DynamicKeywordStatementAst`, `ErrorExpressionAst`, `ErrorStatementAst`, `MemberExpressionAst`, `StringConstantExpressionAst` | `DynamicKeywordPostParseException`, `DynamicKeywordPreParseException`, `MissingBraceInObjectDefinition`, `MissingStatementAfterKeyword`, `RequiredNameOrExpressionMissing`, `UnexpectedNameForType`, `UnexpectedToken`, `UnexpectedTokenInDynamicKeyword` +2 | +| `CreateErrorStatementAst` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CreateErrorStatementAst@L4092-L4098` | — | `ErrorStatementAst` | — | +| `DoWhileStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.DoWhileStatementRule@L4100-L4206` | `Do`, `LParen`, `RParen`, `Until`, `While` | `DoUntilStatementAst`, `DoWhileStatementAst`, `ErrorStatementAst` | `MissingEndParenthesisAfterStatement`, `MissingExpressionAfterKeyword`, `MissingLoopStatement`, `MissingOpenParenthesisAfterKeyword`, `MissingWhileOrUntilInDoWhile` | +| `ClassDefinitionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ClassDefinitionRule@L4208-L4379` | `Colon`, `Comma`, `LCurly`, `RCurly` | `ErrorStatementAst`, `TypeConstraintAst`, `TypeDefinitionAst` | `ClassesNotAllowedInConstrainedLanguage`, `MissingEndCurlyBrace`, `MissingNameAfterKeyword`, `MissingTypeBody`, `TypeNameExpected` | +| `ClassMemberRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ClassMemberRule@L4381-L4639` | `Equals`, `Hidden`, `NewLine`, `Private`, `Public`, `RCurly`, `Semi`, `Static`, `Variable` | `FunctionMemberAst`, `PropertyMemberAst` | `DuplicateQualifier`, `IncompleteMemberDefinition`, `MissingPropertyTerminator`, `ModifiersCannotBeCombined`, `TooManyTypes` | +| `TryUseTokenAsSimpleName` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.TryUseTokenAsSimpleName@L4641-L4652` | `DynamicKeyword`, `Identifier` | — | — | +| `RecordErrorAsts` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.RecordErrorAsts@L4654-L4664` | — | — | — | +| `RecordErrorAsts` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.RecordErrorAsts@L4666-L4676` | — | — | — | +| `NextTypeIdentifierToken` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.NextTypeIdentifierToken@L4678-L4699` | `Identifier` | — | — | +| `EnumDefinitionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.EnumDefinitionRule@L4701-L4820` | `Colon`, `LCurly`, `RCurly` | `ErrorStatementAst`, `TypeConstraintAst`, `TypeDefinitionAst` | `InvalidUnderlyingType`, `MissingEndCurlyBrace`, `MissingNameAfterKeyword`, `MissingTypeBody`, `TypeNameExpected` | +| `EnumMemberRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.EnumMemberRule@L4822-L4895` | `Equals`, `NewLine`, `RCurly`, `Semi` | `PropertyMemberAst` | `ExpectedValueExpression`, `MissingPropertyTerminator` | +| `UsingStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.UsingStatementRule@L4897-L5086` | `Assembly`, `Comma`, `Command`, `EndOfInput`, `Equals`, `Module`, `Namespace`, `NewLine`, `Semi`, `Type` | `ErrorStatementAst`, `UsingStatementAst` | `InvalidValueForUsingItemName`, `MissingEqualsInUsingAlias`, `MissingNamespaceAlias`, `MissingUsingItemName`, `MissingUsingStatementDirective`, `UnexpectedUnaryOperator` | +| `ResolveUsingAssembly` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168` | — | `StringConstantExpressionAst` | `CannotLoadAssemblyFromUncPath`, `CannotLoadAssemblyWithUriSchema`, `ErrorLoadingAssembly` | +| `MethodDeclarationRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.MethodDeclarationRule@L5170-L5314` | `Base`, `Colon`, `LCurly`, `LParen` | `BaseCtorInvokeMemberExpressionAst`, `CommandExpressionAst`, `ErrorStatementAst`, `FunctionDefinitionAst` | `MissingBaseCtorCall`, `MissingFunctionBody`, `MissingMethodParameterList` | +| `FunctionDeclarationRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.FunctionDeclarationRule@L5316-L5413` | `Ampersand`, `AndAnd`, `AtCurly`, `AtParen`, `EndOfInput`, `Filter`, `Generic`, `HereStringExpandable`, `HereStringLiteral`, `LCurly`, `LParen`, `OrOr`, `Pipe`, `RCurly` +9 | `ErrorStatementAst`, `FunctionDefinitionAst` | `MissingFunctionBody`, `MissingNameAfterKeyword` | +| `FunctionParameterDeclarationRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.FunctionParameterDeclarationRule@L5415-L5447` | `LParen`, `RParen` | — | `MissingEndParenthesisInFunctionParameterList` | +| `TrapStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.TrapStatementRule@L5449-L5481` | — | `ErrorStatementAst`, `TrapStatementAst` | `MissingTrapStatement` | +| `CatchBlockRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CatchBlockRule@L5494-L5591` | `Catch`, `Comma` | `CatchClauseAst` | `MissingCatchHandlerBlock`, `MissingTypeLiteralToken` | +| `TryStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.TryStatementRule@L5593-L5670` | `Finally` | `ErrorStatementAst`, `TryStatementAst` | `MissingCatchOrFinally`, `MissingFinallyStatementBlock`, `MissingTryStatementBlock` | +| `DataStatementRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.DataStatementRule@L5672-L5767` | `Comma`, `Parameter` | `DataStatementAst`, `ErrorStatementAst` | `InvalidParameterForDataSectionStatement`, `MissingStatementBlockForDataSection`, `MissingValueForSupportedCommandInDataSectionStatement` | +| `PipelineChainRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988` | `Ampersand`, `AndAnd`, `Dot`, `EndOfInput`, `OrOr` | `AssignmentStatementAst`, `ErrorStatementAst`, `PipelineChainAst` | `BackgroundOperatorInPipelineChain`, `EmptyPipelineChainElement`, `ExpectedValueExpression` | +| `PipelineRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineRule@L5990-L6160` | `Ampersand`, `AndAnd`, `EndOfInput`, `NewLine`, `OrOr`, `Pipe`, `RCurly`, `RParen`, `Semi` | `CommandExpressionAst`, `PipelineAst` | `EmptyPipeElement`, `ExpressionsMustBeFirstInPipeline`, `UnexpectedToken` | +| `RedirectionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.RedirectionRule@L6162-L6264` | — | `ErrorExpressionAst`, `FileRedirectionAst`, `MergingRedirectionAst` | `MissingFileSpecification`, `RedirectionNotSupported`, `StreamAlreadyRedirected` | +| `GetSingleCommandArgument` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetSingleCommandArgument@L6266-L6283` | `Comma`, `EndOfInput` | — | — | +| `GetCommandArgument` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetCommandArgument@L6295-L6442` | `Ampersand`, `AndAnd`, `AtCurly`, `AtParen`, `Comma`, `DollarParen`, `EndOfInput`, `Generic`, `HereStringExpandable`, `HereStringLiteral`, `LCurly`, `LParen`, `MinusMinus`, `NewLine` +12 | `ArrayLiteralAst`, `ErrorExpressionAst`, `ExpandableStringExpressionAst`, `StringConstantExpressionAst` | `MissingExpression` | +| `CommandRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CommandRule@L6444-L6684` | `Ampersand`, `AndAnd`, `Comma`, `Dot`, `EndOfInput`, `InlineScript`, `MinusMinus`, `NewLine`, `OrOr`, `Parameter`, `Pipe`, `RCurly`, `RParen`, `RedirectInStd` +3 | `CommandAst`, `CommandParameterAst`, `DynamicKeywordStatementAst`, `StringConstantExpressionAst` | `MissingArgument`, `MissingExpression`, `ParameterRequiresArgument` | +| `ExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpressionRule@L6696-L6790` | `Colon`, `QuestionMark` | `ErrorExpressionAst`, `TernaryExpressionAst` | `ExpectedValueExpression`, `MissingColonInTernaryExpression` | +| `BinaryExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950` | `AndAnd`, `OrOr` | `BinaryExpressionAst`, `ErrorExpressionAst` | `ExpectedValueExpression` | +| `ErrorRecoveryParameterInExpression` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ErrorRecoveryParameterInExpression@L6952-L6967` | — | `CommandParameterAst`, `ErrorExpressionAst` | `UnexpectedToken` | +| `ArrayLiteralRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ArrayLiteralRule@L6975-L7021` | `Comma` | `ArrayLiteralAst`, `ErrorExpressionAst` | `MissingExpressionAfterToken` | +| `UnaryExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.UnaryExpressionRule@L7029-L7226` | `Comma`, `Generic`, `LBracket`, `Minus`, `MinusMinus`, `NewLine`, `PlusPlus`, `PostfixMinusMinus`, `PostfixPlusPlus`, `Unknown` | `ArrayLiteralAst`, `AttributedExpressionAst`, `ConvertExpressionAst`, `ErrorExpressionAst`, `TypeExpressionAst`, `UnaryExpressionAst` | `MissingExpressionAfterOperator`, `UnexpectedAttribute` | +| `PrimaryExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.PrimaryExpressionRule@L7228-L7299` | `AtCurly`, `AtParen`, `DollarParen`, `HereStringExpandable`, `HereStringLiteral`, `LCurly`, `LParen`, `Number`, `SplattedVariable`, `StringExpandable`, `StringLiteral`, `Variable` | `ConstantExpressionAst`, `StringConstantExpressionAst` | — | +| `CheckUsingVariable` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CheckUsingVariable@L7301-L7317` | `SplattedVariable` | `UsingExpressionAst`, `VariableExpressionAst` | — | +| `CheckPostPrimaryExpressionOperators` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.CheckPostPrimaryExpressionOperators@L7319-L7339` | `ColonColon`, `Dot`, `LBracket`, `QuestionDot`, `QuestionLBracket` | — | — | +| `HashExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.HashExpressionRule@L7341-L7403` | `NewLine`, `RCurly`, `Semi` | `HashtableAst` | `IncompletePropertyAssignmentBlock`, `MissingEndCurlyBrace` | +| `GetKeyValuePair` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetKeyValuePair@L7405-L7491` | `Equals`, `NewLine`, `RCurly`, `Semi` | `ErrorStatementAst` | `MissingEqualsInHashLiteral`, `MissingEqualsInPropertyAssignmentBlock`, `MissingStatementInHashLiteral` | +| `ScriptBlockExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ScriptBlockExpressionRule@L7493-L7517` | — | `ScriptBlockExpressionAst` | — | +| `SubExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SubExpressionRule@L7519-L7571` | `AtParen`, `DollarParen`, `RParen` | `ArrayExpressionAst`, `StatementBlockAst`, `SubExpressionAst` | `MissingEndParenthesisInSubexpression` | +| `ParenthesizedExpressionRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParenthesizedExpressionRule@L7573-L7619` | `RParen` | `ErrorStatementAst`, `ParenExpressionAst` | `ExpectedExpression`, `MissingEndParenthesisInExpression` | +| `ParseNestedExpressions` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseNestedExpressions@L7621-L7672` | `Unknown` | `StringConstantExpressionAst` | — | +| `ExpandableStringRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpandableStringRule@L7674-L7692` | — | `ExpandableStringExpressionAst`, `StringConstantExpressionAst` | — | +| `MemberNameRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.MemberNameRule@L7694-L7716` | `LBracket` | — | — | +| `MemberAccessRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.MemberAccessRule@L7718-L7775` | `ColonColon`, `LCurly`, `LParen`, `QuestionDot` | `ErrorExpressionAst`, `MemberExpressionAst` | `MissingPropertyName` | +| `GenericMethodArgumentsRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.GenericMethodArgumentsRule@L7777-L7847` | `Identifier`, `LBracket`, `RBracket` | — | `EndSquareBracketExpectedAtEndOfType` | +| `MemberInvokeRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.MemberInvokeRule@L7849-L7887` | `ColonColon`, `LParen`, `QuestionDot` | `InvokeMemberExpressionAst` | — | +| `InvokeParamParenListRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.InvokeParamParenListRule@L7889-L7968` | `Comma`, `RParen` | — | `MissingEndParenthesisInMethodCall`, `MissingExpressionAfterToken` | +| `ElementAccessRule` | rule | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ElementAccessRule@L7970-L8020` | `QuestionLBracket`, `RBracket` | `ErrorExpressionAst`, `IndexExpressionAst` | `MissingArrayIndexExpression`, `MissingEndSquareBracket` | +| `SaveError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8044` | — | — | — | +| `SaveError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8046-L8057` | — | — | — | +| `AssertErrorIdCorrespondsToMsgString` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.AssertErrorIdCorrespondsToMsgString@L8067-L8093` | — | — | — | +| `ReportIncompleteInput` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8111-L8118` | — | — | — | +| `ReportIncompleteInput` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8120-L8128` | — | — | — | +| `ReportIncompleteInput` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8130-L8141` | — | — | — | +| `ReportError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8143-L8146` | — | — | — | +| `ReportError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8148-L8152` | — | — | — | +| `ReportError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8154-L8159` | — | — | — | +| `ReportError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8161-L8164` | — | — | — | +| `ReportError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8166-L8169` | — | — | — | +| `ReportErrorsAsWarnings` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportErrorsAsWarnings@L8171-L8184` | — | — | — | +| `ParseError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParseError.ParseError@L8201-L8204` | — | — | — | +| `ParseError` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParseError.ParseError@L8206-L8216` | — | — | — | +| `ToString` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParseError.ToString@L8221-L8224` | — | — | — | +| `ParseStart` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.ParseStart@L8254-L8254` | — | — | — | +| `ParseStop` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.ParseStop@L8256-L8256` | — | — | — | +| `ResolveSymbolsStart` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.ResolveSymbolsStart@L8258-L8258` | — | — | — | +| `ResolveSymbolsStop` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.ResolveSymbolsStop@L8260-L8260` | — | — | — | +| `SemanticChecksStart` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.SemanticChecksStart@L8262-L8262` | — | — | — | +| `SemanticChecksStop` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.SemanticChecksStop@L8264-L8264` | — | — | — | +| `CheckSecurityStart` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.CheckSecurityStart@L8266-L8266` | — | — | — | +| `CheckSecurityStop` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.CheckSecurityStop@L8268-L8268` | — | — | — | +| `CompileStart` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.CompileStart@L8270-L8270` | — | — | — | +| `CompileStop` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.CompileStop@L8272-L8272` | — | — | — | +| `GetFileOrScript` | helper | `src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource.GetFileOrScript@L8274-L8277` | — | — | — | + + +Columns come from `tools/gen_T22.py` by regex over each method body (`APPROX`): tokens are every `TokenKind.X` mentioned, AST names are every `new …Ast(`, error IDs are every `nameof(ParserStrings.X)`. A token that is only compared or skipped is also listed; read the method for the real grammar (see [GRAMMAR.ebnf.md](GRAMMAR.ebnf.md), which is verified by reading). + +### 7.2 Binary operators: precedence climbing + +`BinaryExpressionRule` parses an operand with `ArrayLiteralRule` (so the comma operator binds **tighter than every binary operator**), then repeatedly: peek a token with the `BinaryOperator` trait (but not `&&`/`||`), skip newlines, parse the next operand, and reduce while the new operator's precedence is **less than or equal to** the stack top's. Because the test is `<=`, **all binary operators are left-associative**, including `-and`, `-or`, comparison operators, `??`, `-f` and `..` (`FACT-SRC`, `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950`, loop at L6869-L6937). Precedence is the `BinaryPrecedence*` value in the kind's static flags (`src/System.Management.Automation/engine/parser/token.cs::TokenTraits.GetBinaryPrecedence@L1198-L1205`). A parameter token right after an operand (`1 -foo`) yields `UnexpectedToken` with an `ErrorExpressionAst` (`@L6952-L6967`). The `?:` ternary is handled above this level (§7.3). + +Generated from `TokenFlags` (`GENERATED`, 52 binary operator kinds; higher level binds tighter): + + +| Level (value) | Level name | TokenKind | Text | Also unary | Constant-folded | Restricted-mode flag | +|---:|---|---|---|---|---|---| +| 1 | Logical | `And` | `-and` | | CanConstantFold | | +| 1 | Logical | `Or` | `-or` | | CanConstantFold | | +| 1 | Logical | `Xor` | `-xor` | | CanConstantFold | | +| 2 | Bitwise | `Band` | `-band` | | CanConstantFold | | +| 2 | Bitwise | `Bor` | `-bor` | | CanConstantFold | | +| 2 | Bitwise | `Bxor` | `-bxor` | | CanConstantFold | | +| 5 | Comparison | `As` | `-as` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Ccontains` | `-ccontains` | | | | +| 5 | Comparison | `Ceq` | `-ceq` | | | | +| 5 | Comparison | `Cge` | `-cge` | | | | +| 5 | Comparison | `Cgt` | `-cgt` | | | | +| 5 | Comparison | `Cin` | `-cin` | | | | +| 5 | Comparison | `Cle` | `-cle` | | | | +| 5 | Comparison | `Clike` | `-clike` | | | | +| 5 | Comparison | `Clt` | `-clt` | | | | +| 5 | Comparison | `Cmatch` | `-cmatch` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Cne` | `-cne` | | | | +| 5 | Comparison | `Cnotcontains` | `-cnotcontains` | | | | +| 5 | Comparison | `Cnotin` | `-cnotin` | | | | +| 5 | Comparison | `Cnotlike` | `-cnotlike` | | | | +| 5 | Comparison | `Cnotmatch` | `-cnotmatch` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Creplace` | `-creplace` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Csplit` | `-csplit` | Unary | | DisallowedInRestrictedMode | +| 5 | Comparison | `Icontains` | `-icontains` | | | | +| 5 | Comparison | `Ieq` | `-eq` | | | | +| 5 | Comparison | `Ige` | `-ge` | | | | +| 5 | Comparison | `Igt` | `-gt` | | | | +| 5 | Comparison | `Iin` | `-iin` | | | | +| 5 | Comparison | `Ile` | `-le` | | | | +| 5 | Comparison | `Ilike` | `-ilike` | | | | +| 5 | Comparison | `Ilt` | `-lt` | | | | +| 5 | Comparison | `Imatch` | `-imatch` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Ine` | `-ne` | | | | +| 5 | Comparison | `Inotcontains` | `-inotcontains` | | | | +| 5 | Comparison | `Inotin` | `-inotin` | | | | +| 5 | Comparison | `Inotlike` | `-inotlike` | | | | +| 5 | Comparison | `Inotmatch` | `-inotmatch` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Ireplace` | `-ireplace` | | | DisallowedInRestrictedMode | +| 5 | Comparison | `Is` | `-is` | | | | +| 5 | Comparison | `IsNot` | `-isnot` | | | | +| 5 | Comparison | `Isplit` | `-isplit` | Unary | | DisallowedInRestrictedMode | +| 5 | Comparison | `Join` | `-join` | Unary | | DisallowedInRestrictedMode | +| 5 | Comparison | `Shl` | `-shl` | | CanConstantFold | | +| 5 | Comparison | `Shr` | `-shr` | | CanConstantFold | | +| 7 | Coalesce | `QuestionQuestion` | `??` | | | | +| 9 | Add | `Minus` | `-` | Unary | CanConstantFold | | +| 9 | Add | `Plus` | `+` | Unary | CanConstantFold | | +| 10 | Multiply | `Divide` | `/` | | CanConstantFold | | +| 10 | Multiply | `Multiply` | `*` | | CanConstantFold | | +| 10 | Multiply | `Rem` | `%` | | CanConstantFold | | +| 12 | Format | `Format` | `-f` | | | DisallowedInRestrictedMode | +| 13 | Range | `DotDot` | `..` | | | DisallowedInRestrictedMode | + + +Notes (`FACT-SRC`): comparison, `-as`, `-is`, `-join` (binary form), `-split` forms and `-shl`/`-shr` all share one level (5), so `-shl` binds like `-eq`; `??` is its own level (7) above comparison and below `+`; `-f` (12) binds tighter than `*` (10), and `..` (13) tighter than `-f`; `-and`/`-or`/`-xor` share level 1 so `$a -or $b -and $c` groups left to right. The single authoritative operator table is `OPERATOR-TABLE.md` (planned) (CF-OP-001, planned T28); this section is the parser's view. + +### 7.3 Ternary, unary, array literal, postfix + +- Ternary: `ExpressionRule` parses a binary expression, then `?` (newlines skipped), a full `ExpressionRule` for the true branch (with `endNumberOnTernaryOpChars` so `1?2:3`-style numbers end at `?`/`:`), `:` (`MissingColonInTernaryExpression`), a full `ExpressionRule` for the false branch; right-nested (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpressionRule@L6696-L6790`). The in-source grammar comment says the branches are `ternary-expression`; the code uses the full expression (comment vs code, recorded as CON-004). +- `ArrayLiteralRule`: `unary (',' unary)*`; skipped when `_disableCommaOperator` is set (parameter defaults, method-call arguments) (`@L6975-L7021`). +- `UnaryExpressionRule`: turns on `AllowSignedNumbers`; unary operators (`,` `-not` `!` `-bnot` `+` `-` `++` `--` `-split` `-join`) take a unary operand with newlines allowed after the operator; `[` starts attributes/type constraints → `AttributedExpressionAst`, `ConvertExpressionAst` (cast) or `TypeExpressionAst` (followed by `::`); postfix `++`/`--` apply once to a primary expression (`@L7029-L7226`). +- `PrimaryExpressionRule`: variable (with `$using:` → `UsingExpressionAst`), number, strings (expandable ones parse their nested `$()`/`${}` via a nested tokenizer scan, `ParseNestedExpressions`), `(`, `$(`, `@(`, `@{`, `{`, then a loop of **postfix operators**: `.`, `::`, `?.` member access, `[`/`?[` element access (`@L7228-L7299`, `@L7621-L7672`). +- Member access has **no whitespace** before the operator or the `(`/`{` of an invocation (tokenizer-enforced); `$x.Method{ ... }` passes a script block; generic method arguments `.M[T1,T2]( args )` use TypeName mode and fall back if no `(` follows (`@L7694-L7775`, `@L7777-L7887`). +- Hashtable `@{ ... }`: entries separated by newline or `;`; **the value is a statement** parsed with `StatementRule` in Command mode (`@L7341-L7491`). + +### 7.4 Commands + +`CommandRule` (Command mode) reads elements until `|`, `}`, `)`, end of input, newline, `;`, `&&`, `||`, `&`. A leading `&` or `.` sets the invocation operator and the name may then be an expression. Per element: a `Parameter` token becomes `CommandParameterAst` (with the argument if it ended in `:`); a generic or other token becomes an argument via `GetCommandArgument` (barewords are `StringConstantExpressionAst` with `BareWord` quote type; a `Keyword` flag is cleared so keywords as arguments are plain words); comma-separated arguments become an `ArrayLiteralAst`; redirections become `RedirectionAst`; `--` and `--%` (verbatim) switch to raw handling (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.CommandRule@L6444-L6684`, `Parser.GetCommandArgument@L6295-L6442`, `Parser.RedirectionRule@L6162-L6264`). Never allowed as argument start: `|` `}` `)` end-of-input newline `;` redirection `&&` `||` `&` `--` `,`. + +### 7.5 Keyword statements (summary) + +`if/elseif/else` (`@L2400-L2541`; restore point after the last `}` preserves newlines for hashtable contexts), `switch` with flags `-regex -wildcard -exact -casesensitive -parallel -file` matched by prefix (`@L2543-L2891`), `foreach` (`-parallel`, `-throttlelimit`), `for`, `while`, `do … while|until` (`@L3358-L4206`), `try/catch/finally` (`@L5593-L5670`, catch type lists), `trap`, `data`, `function/filter/workflow` (name tokens that are punctuation, strings or variables are rejected, `@L5316-L5413`), `class` / `enum` / `using` / `configuration` (§14), dynamic keywords (`@L3724-L4090`). Detail per rule is in [GRAMMAR.ebnf.md](GRAMMAR.ebnf.md) with the method link for each production. + +## 8. Observable behavior [SPEC] + +No golden sample: STATIC-ONLY. Behaviors read from source: (a) `$x = 1 | f` parses as an assignment whose right side is a pipeline; (b) `a && b || c` is left-nested; (c) `1 2` is an error but `if ($x) {1} 2` is not; (d) `a |` followed by newline continues; (e) `"a" | % { $_ }` starts a pipeline with an expression; (f) `-and` chains group left to right. Parser tests: `test/powershell/Language/Parser/` (`data/tests.json`); mapping `UNVERIFIED` (VB-005). + +## 9. Error handling and edge cases + +CF-ERR-002 (recovery and limits) lives in [ERROR-CATALOG.md](ERROR-CATALOG.md); the recovery tools are: pretend-present closing tokens (`MissingEndCurlyBrace` etc.), `SyncOnError(consumeClosingToken, syncTokens)` — skips tokens tracking `()`/`{}`/`[]` nesting until a closer or sync token, never consuming end of input (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SyncOnError@L366-L432`) — error AST nodes, and eating a stray token at top level. `ReportIncompleteInput` marks errors at end-of-script so a REPL can request more input (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8111-L8141`). `RuntimeHelpers.EnsureSufficientExecutionStack()` guards the recursive rules (e.g. `StatementRule`, `PipelineChainRule`, `ExpressionRule`, `UnaryExpressionRule`); an `InsufficientExecutionStackException` retries the parse once on a new thread (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`). Comment-vs-code mismatches recorded: `StatementRule` doc says it never returns null but returns null at end of input (CON-005). + +## 10. Concurrency and threading + +A `Parser` instance is single-threaded (own `Tokenizer`, own error list). The argument arrays reused by `ReportError`/`ReportIncompleteInput` are `[ThreadStatic]` fields (`src/System.Management.Automation/engine/parser/Parser.cs::Parser@L8100-L8110`). The stack-overflow retry runs the parse on another thread and blocks (`task.Wait()`). The dynamic-keyword table is thread-local ([02.1-tokenizer.md](02.1-tokenizer.md) §10). + +## 11. Extension points and customization hooks + +New statement keyword: add a `TokenKind` + keyword table row (tokenizer), a `case` in `StatementRule`, a `*Rule` method, an AST class, semantic checks. New operator: tokenizer tables + `TokenFlags` precedence; `BinaryExpressionRule` needs no change for a binary operator with an existing level. Dynamic keywords (DSC/custom) plug in through `DynamicKeyword.PreParse/PostParse/SemanticCheck`. Recipes: HOWTO-extend-language (planned T210). + +## 12. Dependencies + +Inbound: `ScriptBlock`/`CompiledScriptBlock` compile paths, completion, `LanguagePrimitives` (via `ScanNumber`/`ScanType`), module analysis. Outbound: `Tokenizer`, AST classes, `SemanticChecks` (`PerformPostParseChecks`), `ParserStrings`, `DynamicKeyword`, `SystemPolicy`, `Runspace`, DSC subsystem. + +## 13. Performance notes + +One-token lookahead means mode changes re-scan at most one token. Binary-expression parsing is a loop (stack depth does not grow with the number of operators), but nesting (parentheses, blocks) is recursive and guarded. `INFERENCE`: error de-duplication in `SaveError` is linear in the error count per new error. + +## 14. Security considerations (mandatory) + +The parser is not a security boundary, but it has **parse-time side effects and policy checks** (all `FACT-SRC`): + +- `class`: in Constrained Language Mode the parser refuses `class` (`ClassesNotAllowedInConstrainedLanguage`) unless the lockdown policy is `Audit`, in which case it logs a WDAC audit message and continues (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ClassDefinitionRule@L4208-L4379`, L4226-L4244). +- `configuration`: same gate (`ConfigurationNotAllowedInConstrainedLanguage`), refuses on ARM64 and WinPE, creates a default runspace if none exists, and swaps the engine parser while loading DSC keywords (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ConfigurationStatementRule@L2893-L3322`). +- `using assembly`: the **assembly path is resolved at parse time** (UNC paths and non-`file` URI schemes rejected; relative paths resolved against the script directory or the current location; existence checked) (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168`). `using module` / `using namespace` / `using type` only record the directive here. +- `ParseFile` reads the file through `ExternalScriptInfo`. +- Parse of untrusted text is bounded only by the stack guard: no limit on token count or nesting depth beyond `ScriptTooComplicated`; array-type dimension is limited to 32 (`ArrayHasTooManyDimensions`, `@L1523-L1627`). +- Restricted-language checks are not in the parser: they use token traits (`DisallowedInRestrictedMode`), see [TOKEN-CATALOG.md](TOKEN-CATALOG.md). + +## 15. Enterprise considerations + +Observability: ETW `Microsoft-PowerShell-Parser` events around parse and later stages (`src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource@L8247-L8278`; `GetFileOrScript` logs the file name or the first 256 characters of the script, while `ParserEventSource.MaxScriptLengthToLog = 50` is declared on L8252 and is not referenced in `Parser.cs` — recorded as CON-006). Supportability: error IDs are stable resource keys. Missing: no error-count cap, no cancellation token for long parses. + +## 16. Tests + +`test/powershell/Language/Parser/*` (`data/tests.json`). Per-rule mapping `UNVERIFIED` (VB-005). The parser asserts only in `DEBUG` builds (token rescan equality, `CheckAllParentsSet`). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| expression first, then command, in `PipelineChainRule` | `FACT-SRC` | one entry for assignment, expression pipelines and commands; cost: mode switch and rescan of the first token | +| precedence climbing with level numbers from token flags | `FACT-SRC` | new binary operator = new row, no new method; all operators left-associative | +| comma binds tighter than binary operators | `FACT-SRC` | `1,2 -eq 3` compares an array | +| statements as hashtable values | `FACT-SRC` | `@{ a = if ($x) {1} }` is valid | +| recovery by error nodes and pretended closers | `FACT-SRC` | tools get partial ASTs; many errors per script | +| semantic checks run as a separate pass | `FACT-SRC` | parse stays syntax-only; `ModuleAnalysis` mode skips the pass | +| parse-time side effects (`using assembly`, configuration, class gate) | `FACT-SRC` | parsing is not pure; see §14 | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-PARSER-01** [REPLICATE] (area: NLD-GRM) Hand-written recursive descent with exactly one token of lookahead; the parser, not the tokenizer, owns Command/Expression mode and restores it after each rule. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`. +- **LSN-PARSER-02** [REPLICATE] (area: NLD-GRM) At a statement start try an expression in Expression mode first (assignment target or first pipeline element), then fall back to a command. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988`. +- **LSN-PARSER-03** [REPLICATE] (area: NLD-OP) Parse binary operators with an operand stack and operator stack driven by a per-operator precedence level, all left-associative; comma binds tighter than every binary operator; ternary and assignment sit above. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950`. +- **LSN-PARSER-04** [REPLICATE] (area: NLD-GRM) Do not require a terminator after block statements; skip optional newlines and semicolons between statements, and stop pipelines at newline, `;`, `)`, `}`, `&&`, `||`, `&`. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944`. +- **LSN-PARSER-05** [REPLICATE] (area: NLD-ERR) Collect errors with de-duplication, return error AST nodes and recover by pretending closers or skipping to a sync token; mark errors at end-of-script as incomplete input so the REPL can continue the line. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.SyncOnError@L366-L432`, `Parser.ReportIncompleteInput@L8111-L8141`. +- **LSN-PARSER-06** [ADAPT] (area: NLD-GRM) Keep stack-depth protection but replace the retry-on-new-thread with a fixed nesting limit that reports `ScriptTooComplicated`. Why: same observable error, deterministic and testable. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`. +- **LSN-PARSER-07** [AVOID] (area: NLD-GRM) Do not perform filesystem access or runspace creation while parsing (`using assembly` resolution, `configuration`); resolve such directives in a later phase. Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168`. DEV: DEV-GRM-010 + +## 19. Open questions / UNVERIFIED + +- Exact message shape and ordering of errors for common mistakes (needs `pwsh`). +- Behavior of `Resync` interplay with `_ungotToken` in rarely exercised rules (class member parsing); not exercised by reading. +- Which parser tests assert which production (VB-005). +- Whether any caller depends on `StatementRule` returning null at end of input vs the doc comment (CON-005). + +Verification: all `*Rule` methods were read in ≤300-line windows (notes `.work/notes/src_System.Management.Automation_engine_parser_Parser.cs.md`, W1-W28); re-read for this document: `ParseTask`, `SaveError`, `ReportIncompleteInput`, `SyncOnError`, `StatementListRule`, `PipelineChainRule` entry and exit, all `SetTokenizerMode` call sites; method count and the production table reconciled with `data/parser-methods.json` (`PARSER_METHODS`); precedence table generated from `token.cs` static flags and cross-checked against the shunting-yard loop; binary-operator count registered as `BINARY_OPERATORS`. diff --git a/docs-out/02-language/02.3-ast.md b/docs-out/02-language/02.3-ast.md new file mode 100644 index 0000000..9003e72 --- /dev/null +++ b/docs-out/02-language/02.3-ast.md @@ -0,0 +1,203 @@ +--- +id: LANG-AST-001 +title: AST +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/ast.cs, src/System.Management.Automation/engine/parser/AstVisitor.cs, src/System.Management.Automation/engine/parser/PreOrderVisitor.cs, src/System.Management.Automation/engine/parser/ConstantValues.cs, src/System.Management.Automation/engine/parser/SafeValues.cs, src/System.Management.Automation/engine/parser/Position.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/ast-hierarchy.json, data/members-tierA.json, data/visitors.json, data/enums.json] +--- + +# AST + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Every node, its children and its consumers: [AST-CATALOG.md](AST-CATALOG.md) (`GENERATED`). Producer: [02.2-parser.md](02.2-parser.md). Checks run on the tree: [02.4-semantic-analysis.md](02.4-semantic-analysis.md) (planned, T24). + +## 1. Purpose and scope [SPEC] + +The AST is the object tree produced by the parser: one class per syntactic construct, all deriving from `Ast`. It is the interface between parsing and everything else: semantic checks, compilation to expression trees, completion, constant folding, safe-value evaluation, and `ScriptBlock` creation. This document covers the base contract, the visitor infrastructure, search/copy/extent helpers, constant and safe-value evaluation, and the type-name classes that live in `ast.cs`. Node-by-node data is in the catalog. `FACT-SRC` (`src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420`). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| `src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420` | abstract class | base: `Extent`, `Parent`, `Visit`, `Find`/`FindAll`, `Copy`, `SafeGetValue` | A | +| `src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst@L775-L1739` | class | root; named blocks, `param`, using statements, `ScriptRequirements` | A | +| `src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor@L14-L202` | interface | value-returning visitor (default interface methods call `DefaultVisit`) | A | +| `src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235` | interface | adds the nine second-generation node visits | A | +| `src/System.Management.Automation/engine/parser/AstVisitor.cs::AstSearcher@L404-L617` | class | implements `Find`/`FindAll`/`FindFirst` | A | +| `src/System.Management.Automation/engine/parser/AstVisitor.cs::DefaultCustomAstVisitor@L622-L788` | abstract class | no-op base for `ICustomAstVisitor` | A | +| `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215` | abstract class | pre-order visitor returning `AstVisitAction`; post-visit hook | A | +| `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitAction@L12-L28` | enum | `Continue`, `SkipChildren`, `StopVisit` | A | +| `src/System.Management.Automation/engine/parser/ConstantValues.cs::IsConstantValueVisitor@L18-L330` | class | decides whether a node is a compile-time constant | A | +| `src/System.Management.Automation/engine/parser/ConstantValues.cs::ConstantValueVisitor@L332-L577` | class | computes the constant value (by compiling and invoking) | A | +| `src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356` | class | decides whether a node is safe to evaluate from untrusted data | A | +| `src/System.Management.Automation/engine/parser/SafeValues.cs::GetSafeValueVisitor@L363-L806` | class | evaluates safe nodes without executing commands | A | +| `src/System.Management.Automation/engine/parser/Position.cs::PositionUtilities@L114-L385` | static class | extent arithmetic, error-message rendering | A | + +## 3. Responsibilities and non-responsibilities + +Responsible for: node construction invariants (argument checks), parent links, deep copy, child traversal order, search, text round-trip helpers for remoting (`ToStringForSerialization`), and the constant/safe evaluators. Not responsible for: building nodes from tokens (parser), semantic validation (`SemanticChecks`, `SymbolResolver`, `VariableAnalysis`), execution (compiler, [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md), planned T25). + +## 4. Key types and contracts [IMPL] + +**`Ast` contract** (`src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420`): + +- Constructor `protected Ast(IScriptExtent)` throws if the extent is null; `Extent` is get-only; `Parent` has a **private setter** (L108-L118). +- Parent links are set once by the owning node's constructor through `SetParent`/`SetParents`; `SetParent` throws `InvalidOperationException(ParserStrings.AstIsReused)` if the child already has a parent (L267-L300). Constructors that re-home a node call `ClearParent()` first (e.g. `NamedBlockAst` takes the statements out of a `StatementBlockAst`; `AssignmentStatementAst` replaces a one-element pipeline by its `CommandExpressionAst`). +- `Visit(ICustomAstVisitor)` → internal `Accept`; `Visit(AstVisitor)` → internal `InternalVisit` (L126-L140). +- `Find(predicate, searchNestedScriptBlocks)` / `FindAll(...)` → `AstSearcher` (L156-L172). `ToString()` returns `Extent.Text`. `Copy()` is abstract; helpers `CopyElement`/`CopyElements` are null-safe (L185-L264). +- `SafeGetValue(skipHashtableSizeCheck = false)`: evaluates the node with `GetSafeValueVisitor`; **any** exception is rethrown as `InvalidOperationException(CantConvertWithDynamicExpression, Extent.Text)` (L203-L232). +- `IsInWorkflow()` and ancestor helpers walk `Parent` and stop at function boundaries (L313-L417). +- Internal marker interfaces used by the compiler: `ISupportsAssignment`, `IAssignableValue` (compile-time assignment protocol on nodes), `IParameterMetadataProvider` (implemented by `ScriptBlockAst`, `FunctionDefinitionAst`, `FunctionMemberAst`), `ISupportsTypeCaching` (L34-L85, L8416). + +**Counts.** `ast.cs` declares 76 `Ast`-derived classes (11 abstract; catalog §3). `ICustomAstVisitor` has 54 visit methods and `ICustomAstVisitor2` adds 9. + +**Node families** (names are `…Ast`; see catalog for children): + +| Family | Nodes | +|---|---| +| Root and blocks | `ScriptBlockAst`, `ParamBlockAst`, `NamedBlockAst`, `StatementBlockAst` | +| Attributes and parameters | `AttributeBaseAst` → `AttributeAst`, `TypeConstraintAst`; `NamedAttributeArgumentAst`, `ParameterAst` | +| Statements | `StatementAst` → `PipelineBaseAst` → `PipelineAst`, `PipelineChainAst` (via `ChainableAst`), `AssignmentStatementAst`, `ErrorStatementAst`; `CommandBaseAst` → `CommandAst`, `CommandExpressionAst`; `LabeledStatementAst` → `SwitchStatementAst`, `LoopStatementAst` → `ForEach/For/While/DoWhile/DoUntil`; `IfStatementAst`, `TryStatementAst`, `CatchClauseAst`, `TrapStatementAst`, `Break/Continue/Return/Exit/ThrowStatementAst`, `FunctionDefinitionAst`, `DataStatementAst`, `BlockStatementAst`, `UsingStatementAst`, `TypeDefinitionAst`, `ConfigurationDefinitionAst`, `DynamicKeywordStatementAst` | +| Command elements | `CommandElementAst` → `CommandParameterAst`, `ExpressionAst` | +| Redirections | `RedirectionAst` → `FileRedirectionAst`, `MergingRedirectionAst` | +| Expressions | `BinaryExpressionAst`, `UnaryExpressionAst`, `TernaryExpressionAst`, `ConvertExpressionAst`, `AttributedExpressionAst`, `MemberExpressionAst` → `InvokeMemberExpressionAst` → `BaseCtorInvokeMemberExpressionAst`, `IndexExpressionAst`, `ConstantExpressionAst` → `StringConstantExpressionAst`, `ExpandableStringExpressionAst`, `VariableExpressionAst`, `TypeExpressionAst`, `ArrayLiteralAst`, `ArrayExpressionAst`, `HashtableAst`, `ScriptBlockExpressionAst`, `ParenExpressionAst`, `SubExpressionAst`, `UsingExpressionAst`, `ErrorExpressionAst` | +| Class members | `MemberAst` → `PropertyMemberAst`, `FunctionMemberAst`, `CompilerGeneratedMemberFunctionAst` (internal) | +| Not produced by the parser | `SequencePointAst` (compiler only; all visit methods `Assert(false)`), `CompilerGeneratedMemberFunctionAst` | + +## 5. Data model + +- **Immutability.** Public properties are get-only; collections are `ReadOnlyCollection` (never null for statement lists; null when absent for optional single children). One exception: `ArrayLiteralAst` wraps the caller's `IList` without copying (`src/System.Management.Automation/engine/parser/ast.cs::ArrayLiteralAst@L10048-L10123`, `INFERENCE`: caller mutation would be visible). +- **Mutable annotations.** Some nodes carry mutable internal fields filled by later phases: `VariableExpressionAst.TupleIndex/Automatic/Assigned` and `DataStatementAst.TupleIndex` (set by variable analysis, used by the compiler), `UsingExpressionAst.RuntimeUsingIndex`, `TypeDefinitionAst.Type` (null → `TypeBuilder` → runtime type), `ScriptBlockAst.HadErrors/PostParseChecksPerformed/IsConfiguration`, `PipelineAst.Background` (internal setter, set by the parser after construction), `ErrorStatementAst` extras. `INFERENCE`: the tree is therefore immutable in shape but not in annotations; `Copy()` does not copy these caches except where noted below. +- **Tuple children.** `IfStatementAst.Clauses` = `Tuple`, `SwitchStatementAst.Clauses` = `Tuple`, `HashtableAst.KeyValuePairs` = `Tuple` — a hashtable **value is a statement** (matches the parser) (`src/System.Management.Automation/engine/parser/ast.cs::HashtableAst@L10128-L10218`). +- **Normalizations in constructors.** `AssignmentStatementAst` requires the operator kind to carry `AssignmentOperator` and replaces a right side that is a non-background `PipelineAst` with one `CommandExpressionAst` by that expression (`@L6379-L6470`). `ScriptBlockAst` built from a statement block wraps it in an unnamed `NamedBlockAst` (`Process` for filters, otherwise `End`) (`@L1125-L1162`). `NamedBlockAst` rejects block kinds without the `ScriptBlockBlockName` trait (`@L1864-L2026`). +- **Type names are not nodes.** `ITypeName` (`TypeName`, `GenericTypeName`, `ArrayTypeName`, `ReflectionTypeName`) are plain objects held by `TypeConstraintAst`, `TypeExpressionAst`, `AttributeBaseAst`; they resolve to a runtime `Type` lazily (`GetReflectionType`) and cache the result with `Interlocked.CompareExchange` (`@L8369-L9345`). Nested array/generic names are guarded by `EnsureSufficientExecutionStack`. +- **Extents.** Each node has an `IScriptExtent` (start/end line, column, offset, text). Positions are computed lazily from a shared `PositionHelper` line-start table; `ScriptPosition`/`ScriptExtent` are the public, helper-free variants (`src/System.Management.Automation/engine/parser/Position.cs::PositionHelper@L391-L437`, `InternalScriptExtent@L470-L543`). +- **`ScriptRequirements`** and **`CommentHelpInfo`** are plain classes in `ast.cs`, not nodes (`@L713-L770`, `@L10655-L10845`). + +## 6. Control flow + +### 6.1 Two visitor families + +```mermaid +flowchart TD + A[Ast.Visit ICustomAstVisitor] --> B[Accept: VisitX returns object] + C[Ast.Visit AstVisitor] --> D[InternalVisit: pre-order walk] + D --> E[VisitX returns AstVisitAction] + E -->|Continue| F[visit children in order] + E -->|SkipChildren| G[CheckForPostAction then Continue] + E -->|StopVisit| H[CheckForPostAction then stop] + F --> I[CheckForPostAction: IAstPostVisitHandler.PostVisit] +``` + +- **`ICustomAstVisitor` / `ICustomAstVisitor2`** (`Accept`): the visitor decides recursion itself and returns an `object`. Used by the compiler, constant/safe evaluators, variable analysis, type inference. Every interface method has a default body calling `DefaultVisit(ast)` which returns `null` (`src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor@L14-L202`). +- **`AstVisitor` / `AstVisitor2`** (`InternalVisit`): pre-order walk. `VisitX(node)` is called **before** the children; the return value controls the walk (`Continue`, `SkipChildren`, `StopVisit`). `IAstPostVisitHandler.PostVisit(ast)` is called after the children **on every exit path**, including `SkipChildren` and `StopVisit` (`src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor.CheckForPostAction@L35-L43`). +- Canonical `InternalVisit` shape (every node): call `visitor.VisitX(this)`; if `SkipChildren` → return `CheckForPostAction(this, Continue)`; if `Continue` → visit children in fixed order, stopping at the first non-`Continue`; return `CheckForPostAction(this, action)` (`src/System.Management.Automation/engine/parser/ast.cs::ErrorStatementAst@L456-L648`, `IfStatementAst@L4059-L4157`). + +### 6.2 Version gating of nodes added after v1 + +The nine second-generation nodes (`TypeDefinitionAst`, `PropertyMemberAst`, `FunctionMemberAst`, `BaseCtorInvokeMemberExpressionAst`, `UsingStatementAst`, `ConfigurationDefinitionAst`, `DynamicKeywordStatementAst`, `TernaryExpressionAst`, `PipelineChainAst`) are visited only by `AstVisitor2` / `ICustomAstVisitor2`: + +- `Accept` on them casts the visitor to `ICustomAstVisitor2` and **returns `null` for a plain `ICustomAstVisitor`**. +- `InternalVisit` calls their `VisitX` only for `AstVisitor2`. Whether children are still visited for a v1 visitor differs per node: `TypeDefinitionAst`, `TernaryExpressionAst`, `PipelineChainAst` and `BaseCtorInvokeMemberExpressionAst` visit children for every visitor; `PropertyMemberAst`, `FunctionMemberAst` and `UsingStatementAst` visit nothing for a v1 visitor (`UsingStatementAst` also returns `SkipChildren` unchanged to `CheckForPostAction`). Evidence: `src/System.Management.Automation/engine/parser/ast.cs::TypeDefinitionAst@L2667-L2865`, `UsingStatementAst@L2901-L3105`, `PropertyMemberAst@L3156-L3315`, `FunctionMemberAst@L3342-L3630`, `PipelineChainAst@L5549-L5650`, `TernaryExpressionAst@L7404-L7500`. + +The column "v2" in the catalog marks nodes whose `InternalVisit` mentions `AstVisitor2`. + +### 6.3 Search + +`Find`/`FindAll` run `AstSearcher` (a pre-order visitor) starting at the node **itself** (the root is tested first). With `searchNestedScriptBlocks = false` the searcher returns `SkipChildren` after testing a `FunctionDefinitionAst`, `TrapStatementAst` or `ScriptBlockExpressionAst`, so those are tested but not entered; it does **not** skip a nested `ScriptBlockAst` (`src/System.Management.Automation/engine/parser/AstVisitor.cs::AstSearcher@L404-L617`). Cross-reference: `IsUsingDollarInput` looks for `$input` without entering nested script blocks. + +## 7. Algorithms + +### 7.1 Constant folding support (`IsConstantValueVisitor`, `ConstantValueVisitor`) + +`IsConstant(ast, out value, forAttribute, forRequires)`: ask `IsConstantValueVisitor`; if true and the node has a `DataStatementAst` ancestor → **not** constant; otherwise compute the value with `ConstantValueVisitor`; any exception during the computation (e.g. an invalid cast) makes the node non-constant (`src/System.Management.Automation/engine/parser/ConstantValues.cs::IsConstantValueVisitor.IsConstant@L20-L53`). Rules (`FACT-SRC`, L61-L330): only expressions can be constant; `BinaryExpressionAst` needs the `CanConstantFold` token trait on the operator and constant operands and not a divisor that is `$null`/`$false` variable; `UnaryExpressionAst` likewise; `ConvertExpressionAst` needs a resolvable safe primitive type; `$true/$false/$null` variables are constants; a static `const` field of a resolvable type is constant; arrays/hashtables/script blocks are constant only in attribute-argument or `#requires` contexts. **The value is computed by compiling the node and invoking the delegate** (`CompileAndInvoke` with a `Compiler` in `CompilingConstantExpression` mode), so folded operators use the real operator implementations (`src/System.Management.Automation/engine/parser/ConstantValues.cs::ConstantValueVisitor@L332-L577`). + +### 7.2 Safe values (`IsSafeValueVisitor`, `GetSafeValueVisitor`) + +Purpose: evaluate data-only expressions (module manifests, `-Data` sections, argument-completion helpers) without running arbitrary code. `IsSafeValueVisitor` first decides safety: all statement kinds, commands, assignments, member invocations, and **binary expressions** are unsafe (the source comment gives a denial-of-service example, `"AAAA"*2*2…`); type and member expressions are unsafe (they would reveal installed types); unary and convert expressions are safe only when the operator is constant-foldable, not restricted, and the type is a safe primitive; the visit counter is capped at 5000 nodes and hashtables at 500 keys unless `SkipHashtableSizeCheck` is requested (`src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356`, constants in the constructor L47-L53). `GetSafeValueVisitor` then evaluates by itself, except that unary/convert/ternary operands go through the compiler (`Compiler.GetExpressionValue(..., isTrustedInput: true, ...)`); `$PSScriptRoot`, `$true/$false/$null` and—when an execution context exists—ordinary variables are resolved; an expandable string joins arrays with `$OFS` (default space); a script block expression yields a new `ScriptBlock` from its text without invoking it (`src/System.Management.Automation/engine/parser/SafeValues.cs::GetSafeValueVisitor@L363-L806`). Contexts: `Default`, `GetPowerShell`, `ModuleAnalysis`, `SkipHashtableSizeCheck`. + +### 7.3 Copying + +`Copy()` deep-copies children with `CopyElement(s)`; it preserves script-level annotations selectively (`ScriptBlockAst.Copy` keeps `IsConfiguration` and shares `ScriptRequirements`, drops `HadErrors`/`PostParseChecksPerformed`; `UsingExpressionAst` keeps `RuntimeUsingIndex`; `CommandAst` keeps `DefiningKeyword`) (`src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst@L775-L1739`). + +## 8. Observable behavior [SPEC] + +No golden sample: STATIC-ONLY. Observable through the public API: `Parser.ParseInput` returns the root; `Ast.FindAll(pred, true)` enumerates nodes in pre-order; `Ast.ToString()` returns the source text; `ScriptBlockAst.GetScriptBlock()` runs the post-parse checks if needed and throws `ParseException` on errors; `Ast.SafeGetValue()` throws `InvalidOperationException` for anything unsafe. Tests: `test/powershell/Language/Parser/` and `test/xUnit/csharp/test_AstDefaultVisit.cs` (default-visit behavior of the four visitor bases) (`data/tests.json`; per-assertion mapping `UNVERIFIED`, VB-005). + +## 9. Error handling and edge cases + +- Constructors throw `ArgumentNullException`/`ArgumentException` for invalid combinations (e.g. `IfStatementAst` with no clause, `SwitchStatementAst` with no clauses and no default, `TryStatementAst` with neither catch nor finally, `FunctionDefinitionAst` both filter and workflow, `PipelineChainAst` operator not `&&`/`||`, `UnaryExpressionAst`/`BinaryExpressionAst` operator without the matching token trait). +- Re-using a node under two parents throws `AstIsReused`. +- `ErrorStatementAst`/`ErrorExpressionAst` carry what parsed before an error (parser recovery); they are never produced by valid scripts. +- `ScriptBlockAst.GetScriptBlock()` throws if the tree had errors (`PSInvalidOperationException`) or if post-parse checks report errors (`ParseException`) (`src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst.GetScriptBlock@L1250-L1272`). +- Reading observations that look like defects (`INFERENCE`, not run; recorded in the backlog VB-008 to VB-012): `UsingStatementAst.Copy` selects its constructor from `Alias != null` only and loses the module-spec forms (`@L3037-L3045`); `ConfigurationDefinitionAst.InternalVisit` ignores the action returned for its body (`@L6599-L6620`); `BaseCtorInvokeMemberExpressionAst` has no `Copy` override so a copy becomes a plain `InvokeMemberExpressionAst`; `ForEachStatementAst.ThrottleLimit` and `DataStatementAst.CommandsAllowed` are not visited (`@L4469`, `@L4242-L4250`); multi-line `ScriptExtent.Text` indexes the first line with a 1-based column (`src/System.Management.Automation/engine/parser/Position.cs::ScriptExtent@L696-L815`). +- Comment-versus-code: the file header says private/internal methods live in `AstCompile.cs`, which does not exist (CON-007); the `SafeValues.cs` comment says `$OFS` cannot be used but the code reads it from the session state when a context exists (CON-008); the study prompt's hint that `AstVisitor.cs` defines `AstVisitor` is wrong, it is in `PreOrderVisitor.cs`. + +## 10. Concurrency and threading + +AST objects are shared between threads after construction (script blocks are cached and executed concurrently), which works because shape is immutable; annotation fields set by the compiler are not locked (`INFERENCE`). Type resolution caches publish with `Interlocked.CompareExchange`. `GetSafeValueVisitor` keeps its context in a `[ThreadStatic]` field (`t_context`). Lazily built DSC caches in `ConfigurationDefinitionAst` have no lock (out of scope). + +## 11. Extension points and customization hooks + +Subclass `AstVisitor`/`AstVisitor2` (pre-order walk with skip/stop), `DefaultCustomAstVisitor(2)` (return values), or implement `IAstPostVisitHandler` for a post-order hook. Adding a node means: a class, `Copy`, `Accept`, `InternalVisit`, a `VisitX` in both interface families (with a default body), the debug-only `CheckAllParentsSet` visitor, the compiler `VisitX`, and semantic/variable-analysis handling. Recipes: HOWTO-extend-language (planned T210). + +## 12. Dependencies + +Inbound: parser (construction), `SemanticChecks`, `SymbolResolver`, `VariableAnalysis`, `Compiler`, `TypeInferenceVisitor`, completion, module analysis (`ExportVisitor`), debugger (`CheckBreakpointInScript`), `ScriptBlockToPowerShellChecker`, telemetry (`ScriptBlockTelemetry`) — full list in [AST-CATALOG.md](AST-CATALOG.md) §6. Outbound: `TokenKind`/`TokenFlags` (operator validation), `Compiler` (assignment protocol and constant evaluation), `TypeResolver`, `HelpCommentsParser`, `LanguagePrimitives`, `Parser` (`ScanString`, parse of fixed text in DSC helpers). + +## 13. Performance notes + +Parent links make ancestor walks O(depth); `Find` walks the whole tree. Constant evaluation compiles a lambda per folded expression (`INFERENCE`: expensive for many constants, bounded by tiering decisions elsewhere). Type-name resolution is cached per `TypeName`. + +## 14. Security considerations (mandatory) + +- `SafeGetValue` is the sanctioned way to read literal data from untrusted script text: the safe-value visitors forbid commands, assignments, method calls, type and member expressions, binary operators, and cap visits (5000) and hashtable keys (500). Evidence: `src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356`. The comment-documented threats are denial of service (large repeated operations) and type fingerprinting. +- `ModuleAnalysis` context returns `null` instead of throwing for unsafe nodes. +- `ConstantValueVisitor` **does execute** compiled code for constant-foldable operators; safety rests on `CanConstantFold` excluding culture- and context-dependent operators (comment `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L742-L745`). +- `ConfigurationDefinitionAst` synthesizes AST for DSC at parse time; out of scope here. + +## 15. Enterprise considerations + +Observability: `HasSuspiciousContent` flag on nodes (set by security scanning, `src/System.Management.Automation/engine/parser/ast.cs::Ast@L349`) and parent links give tooling stable navigation. Testability: pure data structure; `DEBUG` visitors check parent links and `TypeBuilder` leakage. Missing: no tree version/serialization format, no node ids. + +## 16. Tests + +`test/xUnit/csharp/test_AstDefaultVisit.cs` (default-visit behavior), `test/powershell/Language/Parser/*` (`data/tests.json`). Mapping `UNVERIFIED`. + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| tree is built once with parent links set in constructors | `FACT-SRC` | simple ownership; nodes cannot be shared between trees (must `Copy`) | +| two visitor families (value-returning and pre-order walk with actions) | `FACT-SRC` | flexibility; every new node must be added to four visitor bases | +| second-generation nodes gated by `*Visitor2` | `FACT-SRC` | old external visitors keep compiling; behavior for new nodes differs per node (§6.2) | +| compile-time caches and assignment logic live on nodes | `FACT-SRC` | fast compile; couples AST to compiler and makes nodes not purely syntactic | +| constants are computed by compiling and running the expression | `FACT-SRC` | exact operator semantics, but evaluation needs the engine | +| separate "is it safe" and "evaluate it" visitors | `FACT-SRC` | auditable allow-list for untrusted data | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-AST-01** [REPLICATE] (area: NLD-GRM) One class per construct, get-only properties, constructor invariants that throw on invalid combinations, and parent links set exactly once. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420`. +- **LSN-AST-02** [REPLICATE] (area: NLD-GRM) Provide a pre-order visitor with `Continue`/`SkipChildren`/`StopVisit` and a post-visit hook that fires on every exit path. Evidence: `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215`. +- **LSN-AST-03** [REPLICATE] (area: NLD-GRM) Keep error nodes (`ErrorStatementAst`, `ErrorExpressionAst`) that hold partially parsed children so tooling works on broken scripts. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::ErrorStatementAst@L456-L648`. +- **LSN-AST-04** [REPLICATE] (area: NLD-GRM) Treat hashtable values as statements and keep `PipelineChainAst` left-nested with a plain `PipelineAst` on the right, as the parser produces them. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::HashtableAst@L10128-L10218`, `PipelineChainAst@L5549-L5650`. +- **LSN-AST-05** [ADAPT] (area: NLD-GRM) Use one visitor family with a default `Visit` for new nodes instead of two families gated by "v2" interfaces. Why: the v1/v2 split exists for source compatibility with external visitors, which a new shell does not have. Evidence: `src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235`. +- **LSN-AST-06** [ADAPT] (area: NLD-GRM) Make type names (`ITypeName`) plain syntax objects resolved lazily by the type system, with a cache owned by the resolver. Why: keeps the AST free of reflection state while keeping the observable `[type]` resolution. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::TypeName@L8425-L8724`. +- **LSN-AST-07** [AVOID] (area: NLD-GRM) Do not store compiler caches (`TupleIndex`, `RuntimeUsingIndex`, `TypeDefinitionAst.Type`) or assignment-code-generation methods on AST nodes; keep them in side tables keyed by node. Evidence: `src/System.Management.Automation/engine/parser/ast.cs::VariableExpressionAst@L9404-L9624`. DEV: DEV-AST-020 +- **LSN-AST-08** [ADAPT] (area: NLD-SEC) Provide an allow-list "literal data" evaluator (no commands, no member calls, no binary operators, node and key caps) for manifests and data sections, evaluated without the compiler. Why: same observable safety as `SafeGetValue`, without depending on the compiler for unary/convert nodes. Evidence: `src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356`. + +## 19. Open questions / UNVERIFIED + +- The `INFERENCE` defects in §9 need a run to confirm (VB-008 to VB-012). +- Complete list of external consumers of `Accept`-style visitors in `Compiler.cs` is T25's. +- `HasSuspiciousContent` setters not traced. +- Mapping of `test_AstDefaultVisit.cs` assertions not read. + +Verification: `ast.cs` (10,845 lines), `AstVisitor.cs` (822), `PreOrderVisitor.cs` (264), `ConstantValues.cs` (578), `SafeValues.cs` (807) and `Position.cs` (816) were read in full windows (notes `.work/notes/…ast.cs.md` W1-W36 and the per-file notes); node, base and visitor counts reconciled with `data/ast-hierarchy.json` (file filter `ast.cs`) and `data/visitors.json` via `tools/gen_T23.py --count`; the catalog's children column was spot-checked against `InternalVisit` bodies of `IfStatementAst`, `ScriptBlockAst`, `ErrorStatementAst`, `InvokeMemberExpressionAst`. diff --git a/docs-out/02-language/02.4-semantic-analysis.md b/docs-out/02-language/02.4-semantic-analysis.md new file mode 100644 index 0000000..7685abb --- /dev/null +++ b/docs-out/02-language/02.4-semantic-analysis.md @@ -0,0 +1,300 @@ +--- +id: LANG-SEMANTIC-001 +title: Semantic analysis +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/SemanticChecks.cs, src/System.Management.Automation/engine/parser/SymbolResolver.cs, src/System.Management.Automation/engine/parser/VariableAnalysis.cs, src/System.Management.Automation/engine/parser/TypeResolver.cs, src/System.Management.Automation/engine/parser/PSType.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/types.json, data/members-tierA.json, data/resources.json] +--- + +# Semantic analysis + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Inputs: the AST of [02.3-ast.md](02.3-ast.md); producer of errors in the same list as [ERROR-CATALOG.md](ERROR-CATALOG.md); consumer of the output: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md). + +## 1. Purpose and scope [SPEC] + +After parsing, three kinds of analysis run over the tree: (a) **symbol resolution** (classes, enums, `using module` types, type names), (b) **semantic checks** (rules the grammar cannot express: duplicate parameters, `break` placement, assignment targets, attribute arguments, class members, restricted-language rules for `data` sections), and (c) **variable analysis** (which variables can live in fixed tuple slots; used by the compiler, partly also by the checks). A fourth stage, **type definition** (`TypeDefiner`), also reports parse-style errors but runs only when the root script block is compiled. This document gives the order, the rules, and a generated table of every check with its error ID. `FACT-SRC` (src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514). + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514 | `AstVisitor2`, `IAstPostVisitHandler` | the semantic checker (partial class, one `Visit*` per rule) | A | +| src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454 | `AstVisitor` | allow-list checker for `data` sections, manifests, restricted evaluation | A | +| src/System.Management.Automation/engine/parser/SemanticChecks.cs::DscResourceChecker@L1516-L1805 | static class | checks a class decorated `[DscResource()]` (DSC; out of scope, summarized) | A | +| src/System.Management.Automation/engine/parser/SymbolResolver.cs::SymbolResolver@L275-L729 | `AstVisitor2`, `IAstPostVisitHandler` | scopes, type table, `using module` import, type-name binding | A | +| src/System.Management.Automation/engine/parser/SymbolResolver.cs::SymbolTable@L161-L273 | class | stack of scopes with type/variable tables | A | +| src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847 | `ICustomAstVisitor2` | flow-graph based variable slot allocation and type deduction | A | +| src/System.Management.Automation/engine/parser/VariableAnalysis.cs::FindAllVariablesVisitor@L44-L323 | `AstVisitor` | first pass: collect variable names, decide on pessimizing | A | +| src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolver@L31-L506 | static class | `[type]` name to `Type` resolution | A | +| src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeAccelerators@L844-L979 | static class | built-in and user type accelerators | A | +| src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505 | static class | Reflection.Emit definition of PowerShell classes and enums | A | + +## 3. Responsibilities and non-responsibilities + +Responsible for: reporting rule violations as parse errors, binding type names to classes/types, deciding variable slots. Not responsible for: syntax (parser), evaluating anything (compiler), command resolution (runtime; commands are never resolved at parse time). + +## 4. Key types and contracts [IMPL] + +- **Order.** `Parser.ParseTask` calls `ScriptBlockAst.PerformPostParseChecks(parser)` for `ParseMode.Default`: first `SymbolResolver.ResolveSymbols(parser, ast)`, then `SemanticChecks.CheckAst(parser, ast)`; the semantic pass sets `PostParseChecksPerformed` and `HadErrors` on every script block it leaves (src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst.PerformPostParseChecks@L1416-L1432, src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks.CheckAst@L39-L47, src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks.PostVisit@L1490-L1513). `ParseMode.ModuleAnalysis` skips both (so module analysis tolerates unresolved classes). +- `SemanticChecks` is a pre-order visitor with a post-visit hook: it keeps a stack of enclosing script blocks (`_scopeStack`) and of class-member scopes (`_memberScopeStack`) pushed in `VisitScriptBlock`/`VisitFunctionMember` and popped in `PostVisit`. +- `SymbolResolver` keeps a `SymbolTable` (stack of `Scope` objects of kinds `Type`, `Method`, `Function`, `ScriptBlock`). Entering a scope pre-registers every class/enum defined in it (forward references between classes work); a class property name is registered once per type; duplicate type names report `MemberAlreadyDefined`. +- `VariableAnalysis` exposes `Analyze` (script blocks/functions), `AnalyzeTrap`, `AnalyzeExpression` (default-argument expressions, single expressions) and `AnalyzeMemberFunction`; its result is a `Tuple>`: the `MutableTuple` type for the locals and the name → slot map. + +## 5. Data model + +- `VariableAnalysisDetails` per variable: slot `BitIndex`/`LocalTupleIndex`, deduced `Type`, `Automatic`, `PreferenceVariable`, `Assigned`, associated variable nodes. Results are written back onto `VariableExpressionAst.TupleIndex/Automatic/Assigned` (see [02.3-ast.md](02.3-ast.md) §5). Sentinels: `Unanalyzed = -1`, `ForceDynamic = -2` (variable is accessed by name through the session state instead of a slot) (src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847). +- `Block` = flow-graph node holding the variable nodes it visits plus successor/predecessor links and `throws/returns/unreachable` flags. +- Type resolution caches: `TypeResolutionState` (namespaces from `using namespace`, assemblies, defined types, generic argument count, attribute flag; immutable), a process-wide `TypeCache` keyed by (type name, state), `CoreTypes` accelerators fully supported in Constrained Language Mode. + +## 6. Control flow + +```mermaid +flowchart TD + P[Parser.ParseTask] --> PP[PerformPostParseChecks] + PP --> SR[SymbolResolver.ResolveSymbols] + SR --> SC[SemanticChecks.CheckAst] + SC --> VA1[VariableAnalysis.AnalyzeMemberFunction for class methods] + SC --> RL[RestrictedLanguageChecker for data statements] + SC --> DSC[DscResourceChecker for DscResource classes] + C[Compiler.Compile] --> VA2[VariableAnalysis.Analyze] + C --> TD[TypeDefiner.DefineTypes for root with classes] +``` + +1. Symbol resolution (`SymbolResolver`): enter the root scope with the `TypeResolutionState` built from `using namespace` statements; visit type definitions, functions, script block expressions (each opens a scope); for `using module` load the module at **parse time** via a `PowerShell` instance running `Get-Module -FullyQualifiedName … -ListAvailable` and register its exported classes by simple and `Module.Type` names; bind every type name (type expression, type constraint, attribute) to a local class definition or, failing that, to a CLR type through `TypeResolver`. +2. Semantic checks (`SemanticChecks`): one pre-order walk; each rule in the generated table below. +3. Variable analysis (`VariableAnalysis`): normally run **by the compiler** (not at parse time) for each script block; run at semantic-check time only for class methods (to verify that all paths return). +4. Type definition (`TypeDefiner`): run by `Compiler.GenerateTypesAndUsings` when the root script block is compiled; errors there are collected in a fresh `Parser`, the half-built types are wiped and a `ParseException` is thrown (src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505). The source comment calls this a short-term solution (the checks belong at parse time); recorded as CON-009. + +## 7. Algorithms + +### 7.1 Type-name resolution order (`TypeResolver`) + +For a type name with resolution state `S` (`ResolveTypeNameWithContext`, cache first): (1) script-defined types in the current session-state scope chain (`using type`, classes); (2) built-in type accelerators; (3) assemblies listed in `S` (reporting ambiguity if two assemblies define the name); (4) user type accelerators; then retries with each `using namespace` namespace prefix; two namespaces matching → `AmbiguousTypeReference`. Generic names retry with `` `N`` appended; attribute names retry with `Attribute` appended. Assembly-qualified names use `Type.GetType(FullName)` (then with `System.` prefixed) and require a public type. Evidence: src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolver@L31-L506, src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolutionState@L520-L670, src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeCache@L672-L701. An unresolved type outside a type definition produces **no** parse error; it is resolved again at run time (`SymbolResolver` reports `TypeNotFound` only inside a type definition and not for `[ordered]`). + +### 7.2 Variable slot allocation (`VariableAnalysis`) + +1. `FindAllVariablesVisitor` collects local variable names (unscoped, `local:`, `private:`; `PSItem` is aliased to `_`) and disables optimization for the whole block when it sees `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases/module-qualified names), a dot-sourcing operator, or a `$private:` variable. +2. The constructor seeds the automatic variables (`$_`, `$args`, `$this`, `$input`, `$PSCmdlet`, `$PSBoundParameters`, `$MyInvocation`, `$PSScriptRoot`, `$PSCommandPath`, … in a fixed order) and, for script cmdlets, the preference variables. +3. A flow graph is built per statement kind (loops, `try`, `switch`, `break/continue/return/throw/exit`, `&&`/`||` as short circuits). `try` is modeled conservatively (catch and finally reachable from the block before the `try`), and a pipeline containing any command inside a loop adds pessimistic edges to all enclosing loop exits. +4. Definite-assignment dataflow: a variable gets a fixed slot only if every read is dominated by an assignment; a read that is not yet definitely assigned, a non-local scope (`$global:`, `$script:`, `$using:`, drive-qualified), a type change between assignments, validation/transformation attributes, more than one conversion, an unresolved conversion type, `[ref]`, or a mutable value type (not `SwitchParameter`) forces `ForceDynamic`. +5. Output: tuple type from the deduced slot types and the name → index map (`FinishAnalysis`). When optimization is disabled only automatic variables (and preference variables for script cmdlets) are allocated. +6. `AnyVariablesCouldBeAllScope` (static registry of names known to be `AllScope`) is consulted at run time: if any allocated variable could be `AllScope`, the optimized code is not used (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83, see [TIERING-POLICY.md](TIERING-POLICY.md)). + +## 8. Observable behavior [SPEC] + +Parse-time errors listed below are visible through `Parser.ParseInput` and as `ParseException` from `ScriptBlock.Create`. Examples read from source: duplicate parameter names in a `param` block are an error; `break`/`continue` with a label that matches no enclosing loop is an error only inside a class method; `return` with a value in a `void` method is an error; `$a++` requires a variable or property; `&&`/`||` as binary expression operands outside a pipeline chain give `InvalidEndOfLine`; hashtable literals with duplicate keys are an error (`DuplicateKeyInHashLiteral`); a `finally` block that contains `break`, `continue` or `return` that leaves it is an error (`ControlLeavingFinally`); `$using:` accepts only variable, member and constant-index chains. No golden sample: STATIC-ONLY. Tests: `test/powershell/Language/Parser/*` (`data/tests.json`, mapping `UNVERIFIED`). + +## 9. Error handling and edge cases + +All rules call `Parser.ReportError`, so they share the de-duplication and no-limit behavior of CF-ERR-002 ([ERROR-CATALOG.md](ERROR-CATALOG.md)). `CheckArrayTypeNameDepth` reports `ScriptTooComplicated` for type names nested deeper than 200. `VariableNotLocal` is the only check that reads `TupleIndex`, so it depends on variable analysis having run for that block. `SemanticChecks` marks every ancestor of a risky node (`[Type]` expression, non-constant member name, static member on a non-type) with `HasSuspiciousContent = true`, which later forces script block logging. + +## 10. Concurrency and threading + +`SymbolResolver` keeps its `PowerShell` instance for `using module` in a `[ThreadStatic]` field; `TypeCache` is a process-wide `ConcurrentDictionary` (first writer wins; null results are cached too); `TypeAccelerators.Add` has no lock while readers lock `userTypeAccelerators` (`INFERENCE`: a data race on user accelerators); `VariableAnalysis` keeps a static `ConcurrentDictionary` of known `AllScope` variable names. + +## 11. Extension points and customization hooks + +Dynamic keywords can register a `SemanticCheck` callback (`Func`) run from `VisitDynamicKeywordStatement`. New rule = a `Visit*` override in `SemanticChecks` that calls `ReportError` with a new `ParserStrings` key. + +## 12. Dependencies + +Inbound: `Parser.ParseTask`, `Compiler.Compile` (variable analysis, type definition), `ScriptBlockAst.GetScriptBlock`. Outbound: `IsConstantValueVisitor` (constant attribute arguments), `TypeResolver`, `ExecutionContext`/`SessionState` (type tables), `PowerShell` + `Get-Module` (using module), DSC subsystem, WDAC audit logging (`SystemPolicy`), `Utils`. + +## 13. Performance notes + +`SymbolResolver` pre-registers all types per scope without descending into nested script blocks; `TypeResolver` caches per (name, state). `using module` costs one `Get-Module -ListAvailable` per module at parse time. Variable analysis is linear in nodes plus a dataflow over blocks. + +## 14. Security considerations (mandatory) + +- **Data sections** (`data { }`) are checked by `RestrictedLanguageChecker`: only allow-listed commands, constants, simple operators, no dot-sourcing, no script blocks, no variables except `$PSCulture`, `$PSUICulture`, `$true`, `$false`, `$null` (and the names the caller allows). Operators with the `DisallowedInRestrictedMode` token trait are rejected. In Constrained Language Mode a `data` statement with allowed commands is refused at run time unless the system lockdown policy is `Audit` (then a WDAC audit message is logged) (`CheckDataStatementLanguageModeAtRuntime`). Evidence: src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454. +- `using module` executes a module lookup (`Get-Module -ListAvailable`) while parsing; wildcard module names are rejected; relative paths resolve against the script directory. +- `HasSuspiciousContent` marking feeds forced logging ([02.3-ast.md](02.3-ast.md) §14). +- Type resolution can load assemblies for assembly-qualified names (`ResolveAssemblyQualifiedTypeName`); constrained language restricts `[type]` use elsewhere (runtime). + +## 15. Enterprise considerations + +Observability: ETW `SemanticChecks` start/stop and `ResolveSymbols` events exist (src/System.Management.Automation/engine/parser/Parser.cs::ParserEventSource@L8248-L8278). Supportability: every rule has a stable resource key. Missing: no rule IDs separate from message keys; checks are not individually switchable; several class-definition errors surface at compile time rather than parse time (§6.4). + +## 16. Tests + +`test/powershell/Language/Parser/*` and class tests under `test/powershell/Language/Classes/` (`data/tests.json`; per-rule mapping `UNVERIFIED`, VB-005). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| semantic rules in a separate visitor after parsing | `FACT-SRC` | parser stays syntax-only; rules can be skipped (`ModuleAnalysis`) | +| classes are validated partly at compile time (`TypeDefiner`) | `FACT-SRC` | some class errors appear only when the script block is first compiled | +| slot allocation by dataflow with a safe fallback (`ForceDynamic`) | `FACT-SRC` | most variables are array slots; any doubt falls back to dynamic lookup | +| pessimistic handling of commands in loops and of `try` | `FACT-SRC` | correctness over speed; possible lost optimization | +| unresolved types are not errors outside class definitions | `FACT-SRC` | late binding; typos in `[type]` surface at run time | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-SEM-01** [REPLICATE] (area: NLD-GRM) Keep semantic rules (duplicate parameters, loop-control placement, assignment targets, attribute-argument constants, hashtable duplicate keys, `$using` shape) as a post-parse pass that reports into the parser's error list. Evidence: src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514. +- **LSN-SEM-02** [REPLICATE] (area: NLD-SEC) Validate `data` sections with an allow-list checker and refuse operators flagged as disallowed in restricted mode. Evidence: src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454. +- **LSN-SEM-03** [REPLICATE] (area: NLD-TYP) Resolve type names in a fixed order (script types, accelerators, assemblies, user accelerators, `using namespace` retries) and cache results per (name, state). Evidence: src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolver@L31-L506. +- **LSN-SEM-04** [ADAPT] (area: NLD-SCP) Do slot allocation as a separate pass with a definite-assignment analysis, but start with a simple rule (slot only if assigned before every read, never for non-local scopes) and keep a dynamic-lookup fallback. Why: same observable variable behavior with far less code than the flow-graph analysis. Evidence: src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847. +- **LSN-SEM-05** [ADAPT] (area: NLD-GRM) Report all class-definition errors in the semantic pass instead of at first compile. Why: same error set, earlier and deterministic. Evidence: src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505. +- **LSN-SEM-06** [AVOID] (area: NLD-SEC) Do not run module lookups (`Get-Module -ListAvailable`) while parsing `using module`; record the directive and resolve it in a later phase. Evidence: src/System.Management.Automation/engine/parser/SymbolResolver.cs::SymbolResolver@L275-L729. DEV: DEV-SEM-030 + +## 19. Open questions / UNVERIFIED + +- Detailed semantics of DSC checks (`DscResourceChecker`, configuration rules) are summarized only (out of scope). +- `TypeInferenceVisitor` (completion) is not analyzed here (`EXTENDED`). +- Whether any rule is unreachable (e.g. `NonStaticMemberAccessInStaticMember` paths) is not established. +- Behavior of `VariableAnalysis` on exotic flows (`trap` with `continue`, nested `switch -file`) is read, not run. + +## 20. Generated check table + +Every `nameof(ParserStrings.X)` site in `SemanticChecks.cs`, `SymbolResolver.cs` and `PSType.cs` (`GENERATED`, `APPROX`; member = innermost member containing the site; trigger = the message template of the resource key; the same key can appear at several sites): + + +| Check (member) | Where | Error ID | Trigger (message template) | Runs when | +|---|---|---|---|---| +| `Scope.AddType` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L98` | `MemberAlreadyDefined` | The member '{0}' is already defined. | post-parse, symbol resolution (before semantic checks) | +| `Scope.AddTypeFromUsingModule` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L131` | `MemberAlreadyDefined` | The member '{0}' is already defined. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitAssignmentStatement` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L409` | `MissingTypeInStaticPropertyAssignment` | Cannot assign property, use '{0}{1}'. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitAssignmentStatement` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L417` | `MissingThis` | Cannot assign property, use '{0}{1}'. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitUsingStatement` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L545` | `RequiresArgumentMustBeConstant` | Argument must be constant. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitUsingStatement` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L552` | `RequiresModuleInvalid` | The argument for the Module parameter is not valid. {0} | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitUsingStatement` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L559` | `WildCardModuleNameError` | Wildcard characters are not supported for module names. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitUsingStatement` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L580` | `ModuleNotFoundDuringParse` | Could not find the module '{0}'. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitTypeName` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L642` | `AmbiguousTypeReference` | Type name '{0}' is ambiguous, it could be '{1}' or '{2}'. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitTypeName` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L671` | `CustomAttributeTypeNotFound` | Cannot find the type for custom attribute '{0}'. Make sure that the assembly that contains this type is loaded. | post-parse, symbol resolution (before semantic checks) | +| `SymbolResolver.VisitTypeName` | `src/System.Management.Automation/engine/parser/SymbolResolver.cs@L676` | `TypeNotFound` | Unable to find type [{0}]. | post-parse, symbol resolution (before semantic checks) | +| `SemanticChecks.static` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L77` | `ParameterAttributeArgumentNeedsToBeConstant` | Attribute argument must be a constant. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.static` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L81` | `ParameterAttributeArgumentNeedsToBeConstantOrScriptBlock` | Attribute argument must be a constant or a script block. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckForDuplicateParameters` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L96` | `DuplicateFormalParameter` | Duplicate parameter ${0} in parameter list. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckForDuplicateParameters` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L107` | `VoidTypeConstraintNotAllowed` | [void] cannot be used as a parameter type, or on the left side of an assignment. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitAttribute` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L187` | `AttributeNotAllowedOnDeclaration` | Attribute '{0}' is not valid on this declaration. It is valid on '{1}' declarations only. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitAttribute` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L203` | `PropertyNotFoundForAttribute` | Property '{0}' cannot be found for attribute '{1}'. Specify one of the following properties: {2}. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitAttribute` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L244` | `DuplicateNamedArgument` | Duplicate named arguments '{0}' are not allowed. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitParameter` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L307` | `OrderedAttributeOnlyOnHashLiteralNode` | The {0} attribute can be specified only on a hash literal node. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitParameter` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L319` | `MultipleTypeConstraintsOnMethodParam` | Multiple type constraints are not allowed on a method parameter. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckArrayTypeNameDepth` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L355` | `ScriptTooComplicated` | Processing was stopped because the script is too complex. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitFunctionMember` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L400` | `ParamBlockNotAllowedInMethod` | A param block is not allowed in a class method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitFunctionMember` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L411` | `NamedBlockNotAllowedInMethod` | A named block is not allowed in a class method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitFunctionMember` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L418` | `ConstructorCantHaveReturnType` | A constructor cannot specify a return type. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitFunctionMember` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L427` | `MethodHasCodePathNotReturn` | Not all code path returns value within method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitFunctionDefinition` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L440` | `OnlyOneParameterListAllowed` | The param statement cannot be used if arguments were specified in the function declaration. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitFunctionDefinition` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L451` | `WorkflowNotSupportedInPowerShellCore` | Workflow is not supported in PowerShell 6+. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitSwitchStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L465` | `KeywordParameterReservedForFutureUse` | The {0} '-{1}' parameter is reserved for future use. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitForEachStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L500` | `KeywordParameterReservedForFutureUse` | The {0} '-{1}' parameter is reserved for future use. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitForEachStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L510` | `KeywordParameterReservedForFutureUse` | The {0} '-{1}' parameter is reserved for future use. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitForEachStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L522` | `ThrottleLimitRequiresParallelFlag` | The ThrottleLimit parameter is only supported on foreach statements that use the Parallel parameter. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitTryStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L546` | `EmptyCatchNotLast` | Catch block must be the last catch block. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitTryStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L573` | `ExceptionTypeAlreadyCaught` | Exception type {0} is already handled by a previous handler. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckLabelExists` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L606` | `LabelNotFound` | Label '{0}' not found inside the method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckForFlowOutOfFinally` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L659` | `ControlLeavingFinally` | Flow of control cannot leave a Finally block. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckForReturnStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L710` | `VoidMethodHasReturn` | Invalid return statement within void method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckForReturnStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L719` | `NonVoidMethodMissingReturnValue` | Invalid return statement within non-void method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckAssignmentTarget` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L801` | `VoidTypeConstraintNotAllowed` | [void] cannot be used as a parameter type, or on the left side of an assignment. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckAssignmentTarget` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L812` | `ReferenceNeedsToBeByItselfInTypeConstraint` | Cannot use [ref] with other types in a type constraint. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.CheckAssignmentTarget` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L832` | `AssignmentStatementToAutomaticNotSupported` | Cannot assign automatic variable '{0}' with type '{1}' | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitAssignmentStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L871` | `InvalidLeftHandSide` | The assignment expression is not valid. The input to an assignment operator must be an object that is able to accept assignments, such as a variable or a property. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitBinaryExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L883` | `InvalidEndOfLine` | The token '{0}' is not a valid statement separator in this version. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitUnaryExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L901` | `OperatorRequiresVariableOrProperty` | The '{0}' operator works only on variables or on properties. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitConvertExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L920` | `OrderedAttributeOnlyOnHashLiteralNode` | The {0} attribute can be specified only on a hash literal node. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitConvertExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L948` | `ReferenceNeedsToBeByItselfInTypeSequence` | Cannot have two occurrences of [ref] in a type sequence. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitConvertExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L995` | `ReferenceNeedsToBeLastTypeInTypeConversion` | [ref] can only be the final type in type conversion sequence. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitUsingExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1024` | `InvalidUsingExpression` | Expression is not allowed in a Using expression. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitVariableExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1070` | `SplattingNotPermittedInArgumentList` | Splatted variables like '@{0}' cannot be part of a comma-separated list of arguments. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitVariableExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1077` | `SplattingNotPermitted` | The splatting operator '@' cannot be used to reference variables in an expression. '@{0}' can be used only as an argument to a command. To reference variables in an expression use '${0}'. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitVariableExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1093` | `VariableNotLocal` | Variable is not assigned in the method. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitVariableExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1103` | `NonStaticMemberAccessInStaticMember` | Cannot access the non-static member '{0}' in a static method or initializer of a static property. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitHashtable` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1127` | `DuplicatePropertyInInstanceDefinition` | Duplicate property assignments are not allowed in an instance definition. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitHashtable` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1132` | `DuplicateKeyInHashLiteral` | Duplicate keys '{0}' are not allowed in hash literals. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitAttributedExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1159` | `UnexpectedAttribute` | Unexpected attribute '{0}'. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitBlockStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1174` | `UnexpectedKeyword` | Unexpected keyword '{0}'. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitScriptBlockExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1291` | `ResourceNotDefined` | Undefined DSC resource '{0}'. Use Import-DSCResource to import the resource. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitUsingStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1319` | `UsingStatementNotSupported` | This syntax of the 'using' statement is not supported. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitUsingStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1330` | `InvalidNamespaceValue` | The specified namespace in the 'using' statement contains invalid characters. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitConfigurationDefinition` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1371` | `UnsupportedNamedBlockInConfiguration` | Configuration only supports the End block in its body. Begin, Process and DynamicParam blocks are not allowed in a configuration. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitDynamicKeywordStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1398` | `DynamicKeywordSemanticCheckException` | Exception occurred when performing semantic check for dynamic keyword '{0}' with details '{1}'. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitDynamicKeywordStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1418` | `ConfigurationInvalidPropertyName` | An attribute name for resource '{0}' was found that is not valid. An attribute name must be a simple string, and cannot contain variables or expressions. Replace '{1}' with a simple string. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitDynamicKeywordStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1429` | `InvalidInstanceProperty` | The member '{0}' is not valid. Valid members are '{1}'. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitDynamicKeywordStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1453` | `RegularResourceUsedInMetaConfig` | Regular DSC resource {0} is not allowed in the meta configuration. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitDynamicKeywordStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1460` | `MetaConfigurationUsedInRegularConfig` | Meta Configuration resource {0} is not allowed in the regular configuration. Use meta configuration resources in a configuration with [DscLocalConfigurationManager()] attribute. | post-parse semantic checks (`PerformPostParseChecks`) | +| `SemanticChecks.VisitPropertyMember` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1480` | `TypeNotAllowedForProperty` | The type '{0}' is not allowed on a property. | post-parse semantic checks (`PerformPostParseChecks`) | +| `DscResourceChecker.CheckType` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1574` | `DscResourceMissingSetMethod` | The DSC resource '{0}' is missing a Set method that returns [void] and accepts no parameters. | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckType` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1582` | `DscResourceMissingGetMethod` | The DSC resource '{0}' is missing a Get method that returns [{0}] and accepts no parameters. | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckType` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1590` | `DscResourceMissingTestMethod` | The DSC resource '{0}' is missing a Test method that returns [bool] and accepts no parameters. | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckType` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1598` | `DscResourceMissingDefaultConstructor` | The DSC resource '{0}' has no default constructor. | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckType` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1606` | `DscResourceMissingKeyProperty` | The DSC resource '{0}' must have at least one key property (using the syntax [DscProperty(Key)].) | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckGet` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1694` | `DscResourceInvalidGetMethod` | Invalid Get method. Get method must return [{0}] and accepts no parameters. | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckGet` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1702` | `DscResourceInvalidGetMethod` | Invalid Get method. Get method must return [{0}] and accepts no parameters. | post-parse, only for classes with `[DscResource()]` | +| `DscResourceChecker.CheckKey` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1794` | `DscResourceInvalidKeyProperty` | Invalid key property. The key property must be of [string], signed/unsigned integer, or Enum types. | post-parse, only for classes with `[DscResource()]` | +| `RestrictedLanguageChecker.CheckDataStatementLanguageModeAtRuntime` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1860` | `DataSectionAllowedCommandDisallowed` | Cannot define data section. Definition of additional supported commands is not supported in this language mode. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitScriptBlock` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1905` | `ScriptBlockNotSupportedInDataSection` | Script block literals are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitParamBlock` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1914` | `ParameterDeclarationNotSupportedInDataSection` | Parameter declarations are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.CheckTypeName` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1939` | `TypeNotAllowedInDataSection` | The type {0} is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitAttribute` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1956` | `AttributeNotSupportedInDataSection` | Attributes are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitFunctionDefinition` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L1979` | `FunctionDeclarationNotSupportedInDataSection` | Function declarations are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitTrap` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2002` | `TrapStatementNotSupportedInDataSection` | The Trap statement is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitSwitchStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2011` | `SwitchStatementNotSupportedInDataSection` | The switch statement is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitDataStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2020` | `DataSectionStatementNotSupportedInDataSection` | The data statement is not allowed in restricted language mode or another Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitForEachStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2029` | `ForeachStatementNotSupportedInDataSection` | Foreach statements are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitDoWhileStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2038` | `DoWhileStatementNotSupportedInDataSection` | The Do and While statements are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitForStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2047` | `ForWhileStatementNotSupportedInDataSection` | For and While statements are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitWhileStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2056` | `ForWhileStatementNotSupportedInDataSection` | For and While statements are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitTryStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2072` | `TryStatementNotSupportedInDataSection` | The Try statement is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitBreakStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2081` | `FlowControlStatementNotSupportedInDataSection` | Flow control statements such as Break, Continue, Return, Exit, and Throw are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitContinueStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2090` | `FlowControlStatementNotSupportedInDataSection` | Flow control statements such as Break, Continue, Return, Exit, and Throw are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitReturnStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2099` | `FlowControlStatementNotSupportedInDataSection` | Flow control statements such as Break, Continue, Return, Exit, and Throw are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitExitStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2108` | `FlowControlStatementNotSupportedInDataSection` | Flow control statements such as Break, Continue, Return, Exit, and Throw are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitThrowStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2117` | `FlowControlStatementNotSupportedInDataSection` | Flow control statements such as Break, Continue, Return, Exit, and Throw are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitDoUntilStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2126` | `DoWhileStatementNotSupportedInDataSection` | The Do and While statements are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitAssignmentStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2136` | `AssignmentStatementNotSupportedInDataSection` | Assignment statements are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitCommand` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2159` | `DotSourcingNotSupportedInDataSection` | Dot sourcing is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitCommand` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2173` | `OperatorNotSupportedInDataSection` | The '{0}' operator is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitCommand` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2180` | `CmdletNotInAllowedListForDataSection` | The command '{0}' is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitCommand` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2194` | `CmdletNotInAllowedListForDataSection` | The command '{0}' is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitMergingRedirection` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2218` | `RedirectionNotSupportedInDataSection` | Redirection is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitFileRedirection` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2227` | `RedirectionNotSupportedInDataSection` | Redirection is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitBinaryExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2248` | `OperatorNotSupportedInDataSection` | The '{0}' operator is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitUnaryExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2261` | `OperatorNotSupportedInDataSection` | The '{0}' operator is not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitVariableExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2347` | `VariableReferenceNotSupportedInDataSection` | A variable that cannot be referenced in restricted language mode or a Data section is being referenced. Variables that can be referenced include the following: {0}. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitMemberExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2357` | `PropertyReferenceNotSupportedInDataSection` | Property references are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitInvokeMemberExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2366` | `MethodCallNotSupportedInDataSection` | Method calls are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitScriptBlockExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2396` | `ScriptBlockNotSupportedInDataSection` | Script block literals are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitIndexExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2422` | `ArrayReferenceNotSupportedInDataSection` | Array references are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitAttributedExpression` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2432` | `AttributeNotSupportedInDataSection` | Attributes are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `RestrictedLanguageChecker.VisitBlockStatement` | `src/System.Management.Automation/engine/parser/SemanticChecks.cs@L2442` | `ParallelAndSequenceBlockNotSupportedInDataSection` | Parallel and sequence blocks are not allowed in restricted language mode or a Data section. | `data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections | +| `TypeDefiner.TryConvertArg` | `src/System.Management.Automation/engine/parser/PSType.cs@L46` | `CannotConvertValue` | Cannot convert the value to type "{0}". | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L336` | `SubtypeArray` | Invalid base type '{0}'. Base type cannot be an array. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L347` | `TypeNotFound` | Unable to find type [{0}]. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L357` | `SealedBaseClass` | Cannot inherit from sealed class '{0}'. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L366` | `SubtypeUnclosedGeneric` | Invalid base type '{0}'. Base type cannot be a generic with unspecified parameters. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L401` | `SubtypeArray` | Invalid base type '{0}'. Base type cannot be an array. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L413` | `SubtypeArray` | Invalid base type '{0}'. Base type cannot be an array. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L423` | `TypeNotFound` | Unable to find type [{0}]. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetBaseTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L436` | `InterfaceNameExpected` | '{0}': Interface name expected. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.DefineMembers` | `src/System.Management.Automation/engine/parser/PSType.cs@L589` | `BaseClassNoDefaultCtor` | Base class '{0}' does not contain a parameterless constructor. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.DefineProperty` | `src/System.Management.Automation/engine/parser/PSType.cs@L602` | `MemberAlreadyDefined` | The member '{0}' is already defined. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.CheckForDuplicateOverload` | `src/System.Management.Automation/engine/parser/PSType.cs@L785` | `MemberAlreadyDefined` | The member '{0}' is already defined. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetParameterTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L817` | `TypeNotFound` | Unable to find type [{0}]. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.GetParameterTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L825` | `TypeNotAllowedForParameter` | The type '{0}' is not allowed on a parameter. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.DefineMethod` | `src/System.Management.Automation/engine/parser/PSType.cs@L875` | `StaticConstructorCantHaveParameters` | A static constructor cannot have any parameters. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypeHelper.DefineMethod` | `src/System.Management.Automation/engine/parser/PSType.cs@L909` | `TypeNotFound` | Unable to find type [{0}]. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.Sort` | `src/System.Management.Automation/engine/parser/PSType.cs@L1174` | `CycleInEnumInitializers` | Cannot define enum because of a cycle in the initialization expressions. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.DefineEnum` | `src/System.Management.Automation/engine/parser/PSType.cs@L1227` | `InvalidUnderlyingType` | '{0}' is not a valid underlying type for enums. Expected a builtin integral type (one of byte, sbyte, short, ushort, int, uint, long or ulong) | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.DefineEnum` | `src/System.Management.Automation/engine/parser/PSType.cs@L1250` | `EnumeratorValueOutOfBounds` | Enumerator value is either too large or too small for {0}. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.DefineEnum` | `src/System.Management.Automation/engine/parser/PSType.cs@L1258` | `CannotConvertValue` | Cannot convert the value to type "{0}". | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.DefineEnum` | `src/System.Management.Automation/engine/parser/PSType.cs@L1268` | `EnumeratorValueMustBeConstant` | Enumerator value must be a constant value. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.DefineEnum` | `src/System.Management.Automation/engine/parser/PSType.cs@L1279` | `EnumeratorValueOutOfBounds` | Enumerator value is either too large or too small for {0}. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineEnumHelper.DefineEnum` | `src/System.Management.Automation/engine/parser/PSType.cs@L1288` | `MemberAlreadyDefined` | The member '{0}' is already defined. | compile of the root script block (class/enum definition) | +| `TypeDefiner.DefineTypes` | `src/System.Management.Automation/engine/parser/PSType.cs@L1419` | `TypeCreationError` | Error during creation of type "{0}". Error message: {1} | compile of the root script block (class/enum definition) | + + +Rows: 133 sites, 104 distinct IDs. + +Verification: `SemanticChecks.cs` (2,455 lines), `SymbolResolver.cs` (763), `TypeResolver.cs` (980), `PSType.cs` (1,520) and `VariableAnalysis.cs` (1,848) were read in full windows (notes `.work/notes/src-System.Management.Automation-engine-parser-*.md`); the table is generated, with sites counted by `tools/gen_T24.py --count`; type-resolution order and the variable-analysis rules were re-read from `TypeResolver.ResolveTypeNameWorker` and `VariableAnalysis.FinishAnalysis/AnalyzeBlock/CheckLHSAssignVar` notes. diff --git a/docs-out/02-language/02.5-compiler-and-engine.md b/docs-out/02-language/02.5-compiler-and-engine.md new file mode 100644 index 0000000..0643bfc --- /dev/null +++ b/docs-out/02-language/02.5-compiler-and-engine.md @@ -0,0 +1,197 @@ +--- +id: LANG-ENGINE-001 +title: Compiler and execution engine +kind: subsystem +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs, src/System.Management.Automation/engine/interpreter/LightCompiler.cs, src/System.Management.Automation/engine/interpreter/Interpreter.cs, src/System.Management.Automation/engine/lang/parserutils.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/members-tierA.json, data/instructions.json] +--- + +# Compiler and execution engine + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. Companion documents: [COMPILATION-MAP.md](COMPILATION-MAP.md) (node → compiled form, generated), [IR-SPEC.md](IR-SPEC.md) (the tree contract), [TIERING-POLICY.md](TIERING-POLICY.md) (compile vs interpret), [INSTRUCTION-SET.md](INSTRUCTION-SET.md), [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). Upstream: [02.3-ast.md](02.3-ast.md), [02.4-semantic-analysis.md](02.4-semantic-analysis.md). + +## 1. Purpose and scope [SPEC] + +How a parsed script becomes running code: `ScriptBlock` creation and caching, the AST → expression-tree compiler (`Compiler`), the choice between interpreting and JIT-compiling the tree, and how each language construct is executed (statement output, flow control, errors). PowerShell does **not** tree-walk the AST: it compiles it to an IR first (`FACT-SRC`, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661 is an `ICustomAstVisitor2` that returns `Expression` objects). A labelled `INFERENCE` section (§7.4) states what a tree-walking evaluator must preserve. + +## 2. Source map + +| path | type | role | tier | +|---|---|---|---| +| src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661 | class | AST → expression trees (`Visit*` per node, ~99 helpers) | A | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData@L41-L540 | class | per-script-block data: AST, compiled delegates (optimized and unoptimized), metadata | A | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177 | partial class | creation, cache, `InvokeWithPipeImpl` | A | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552 | class | runs a script block as an advanced function / script cmdlet | A | +| src/System.Management.Automation/engine/parser/Compiler.cs::FunctionContext@L784-L814 | class | per-invocation state read by compiled code | A | +| src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2051 | class | expression tree → instructions | A | +| src/System.Management.Automation/engine/interpreter/Interpreter.cs::Interpreter@L31-L109 | class | stack-machine loop | A | +| src/System.Management.Automation/engine/lang/parserutils.cs::ParserOps@L232-L1624 | static class | runtime helpers for operators (despite the file name; hint mismatch) | A | + +## 3. Responsibilities and non-responsibilities + +Responsible for: compiling named blocks, parameter metadata and attributes, per-statement error handling, output capture, flow-control implementation, sequence points. Not responsible for: command discovery and parameter binding (runtime, T32/T33), binder rules (T26), conversion rules (T28). + +## 4. Key types and contracts [IMPL] + +- `CompiledScriptBlockData.Compile(bool optimized)` is the entry used at run time; it calls `new Compiler().Compile(data, optimize)` once per variant under a lock (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.ReallyCompile@L163-L195), after `PerformSecurityChecks` (AMSI scan for top-level script blocks, suspicious-content flag). +- `Compiler.Compile(data, optimize)`: variable analysis → lambdas → `CompileTree` per named block (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964). +- Script block creation: `ScriptBlock.Create(text)` parses (`ParseException` on errors), caches by (file name, text) in `s_cachedScripts` (cap 1024, cleared completely when exceeded; skipped for scripts with `using module/assembly` or dynamic keywords) and returns a `Clone()` that shares the compiled data (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.CacheScriptBlock@L609-L637). +- `ScriptBlock.InvokeWithPipeImpl` is the one execution entry (functions, scripts, `&`, `.`, `Invoke`) (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.InvokeWithPipeImpl@L935-L959). + +## 5. Data model + +- Per named block: one delegate per variant (`Action`); locals tuple type and creator; name → slot map (optimized only); sequence points array (shared). +- `FunctionContext` per call ([IR-SPEC.md](IR-SPEC.md) §4.2); `MutableTuple` locals; `Pipe` output objects. +- Classes defined by `class`/`enum` live in a dynamic assembly created at compile time of the root block (`TypeDefiner`, [02.4-semantic-analysis.md](02.4-semantic-analysis.md) §6). + +## 6. Control flow + +### 6.1 From text to output + +```mermaid +sequenceDiagram + participant U as caller + participant SB as ScriptBlock + participant P as Parser + participant D as CompiledScriptBlockData + participant C as Compiler + participant L as LightLambda + participant I as Interpreter + U->>SB: Create(text) + SB->>P: ParseInput + P-->>SB: ScriptBlockAst (errors => ParseException) + U->>SB: InvokeWithPipeImpl + SB->>D: Compile(runOptimized) + D->>C: Compile(data, optimize) + C->>C: VariableAnalysis.Analyze, Visit nodes to lambdas + C->>L: CompileTree creates LightLambda per block + SB->>L: clause(functionContext) + alt compiled delegate ready + L->>L: run JIT code + else + L->>I: Run(frame) + L->>L: count calls, maybe queue JIT + end +``` + +`InvokeWithPipeImpl` order (`FACT-SRC`, notes W4): a `Clean` clause is not supported through this entry (`PSNotSupportedException`); a missing clause returns silently; pick the clause; decide `runOptimized` (CF-CMP-003); create the locals tuple (`MakeLocalsTuple`); set `$_`, `$input`, `$this`; set `$PSScriptRoot`/`$PSCommandPath`; apply language-mode switch rules; set up scope (new local scope with the locals tuple, or dot-source scope pushed onto `DottedScopes`); bind arguments (`BindArgumentsForScriptblockInvoke`); build the `FunctionContext`; log start; call the delegate in `try/finally`; restore language mode, pipes, scope, backed-up variables. + +### 6.2 Statement lists and error handling + +`CompileStatementListWithTraps` is the core of statement-level semantics (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268): + +- 0 statements → empty; the statement count feeds the tiering decision. +- With `trap` statements: set `context.PropagateExceptionsToEnclosingStatementBlock = true`, compile each trap into its own lambda, `funcContext.PushTrapHandlers(types, delegates, tuple types)` for the duration, pop in a `finally`. +- Without traps but inside a block that has them: push null handlers so `CheckActionPreference` rethrows instead of handling. +- 1 statement: `try { stmt; $? = true } catch (FlowControlException) { throw } catch (Exception e) { CheckActionPreference(funcContext, e) }`. +- ≥ 2 statements: a dispatch index `stmt` with a `switch` jumping to the label of statement *i*; the same two catch blocks; after `CheckActionPreference` returns, control `goto`s the dispatch, which runs the statement *after* the one that failed. This is how a non-terminating-style error (ErrorAction Continue / handled) lets the script continue. + +### 6.3 Loops + +All loops come from `GenerateWhileLoop` (while, for with continue-action, switch, foreach through `GenerateIteratorStatement`) and `GenerateDoLoop`. Shape (`FACT-SRC`, comments L4763-L4803): `LoopTop: [loop counter] if (condition) { try { check-for-interrupts; body } catch (BreakException be) { if (be.MatchLabel(label)) goto Break; throw } catch (ContinueException ce) { if (ce.MatchLabel(label)) goto Continue; throw } Continue: [continue-action]; goto LoopTop } Break:`. `break`/`continue` to a loop of the *same lambda* compile to direct jumps; otherwise they throw. `foreach` evaluates the pipeline once, builds an `IEnumerator` with `PSEnumerableBinder` (a non-enumerable value is wrapped as a one-element array; a `null` enumerator skips the loop for `foreach`), stores it in `$foreach`, and for each item assigns the loop variable. `switch` saves `$_`, iterates the same way over `$switch`, and **tests every clause for every item** (all matching clauses run) before running `default` if none matched. `a..b` as the enumerated expression uses a range enumerator without materializing an array. + +### 6.4 try / catch / finally / trap / throw + +- `try`: body compiled with its own statement list; `PropagateExceptionsToEnclosingStatementBlock` set to true inside and restored in a `finally`. +- Catch dispatch: a catch-all-only `try` catches `RuntimeException` (`PipelineStoppedException` is rethrown first); typed catches call `ExceptionHandlingOps.FindMatchingHandler(locals, rte, catchTypes, context)` and `switch` on the returned handler index; unresolved catch types are resolved lazily and cached. In a handler `$_` is set to a new `ErrorRecord` wrapping the exception and `context.CurrentExceptionBeingHandled` is set, both restored afterwards. If no handler matches, `CheckActionPreference` runs. +- `finally`: user statements run inside `try { … } finally { RestoreStoppingPipeline }` after `SuspendStoppingPipeline` (so a stopped pipeline can still run cleanup). +- `throw expr` → `ExceptionHandlingOps.ConvertToException`; bare `throw` rethrows `CurrentExceptionBeingHandled`; `exit n` → `ExitException`. +- Leaving a `finally` block with `break`/`continue`/`return`/`throw` is a parse-time error (`ControlLeavingFinally`, [02.4-semantic-analysis.md](02.4-semantic-analysis.md)); `return` inside `try` runs the `finally` first (expression-tree `TryFinally` semantics, `EXTERNAL`). +- `trap`: each trap becomes its own lambda (always JIT-compiled), its type list pushed with `PushTrapHandlers`; a `return`/`continue` inside a trap lambda is implemented by `ReturnException`. + +### 6.5 Statement output rule + +A command or a bare expression statement writes to the **current output pipe** (`funcContext._outputPipe`); assignments, `++`/`--`, and pure `[void]`-style statements write nothing. `$(…)`, `@(…)`, conditions, assignment right-hand sides and hashtable values run with a **temporary pipe** that collects into a list (`CaptureAstResults`), converted to: the value (assignment; `AutomationNull`, one item or `object[]`), a `bool` (conditions, via `PSPipelineResultToBoolBinder`), or the list as an enumerable (`foreach`, `@()`). If the captured statement throws, the already collected output is either flushed to the outer pipe (`AssignmentWithResultPreservation`, used for `( … )` and `$( … )` where `ShouldPreserveOutputInCaseOfException`) or discarded. `return expr` therefore does not "return a value": it writes the value to the output pipe and jumps to the return label, so the function result is everything written to its output pipe; in class methods the value is converted to the declared return type and written to a separate return pipe. + +## 7. Algorithms + +### 7.1 Compile entry + +src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964: set `Optimize`; `_compilingScriptCmdlet = UsesCmdletBinding`; `VariableAnalysis.Analyze(body, disableOptimizations: !optimize, scriptCmdlet)`; look up the `$switch`/`$foreach` slots; build the locals parameter; for class members set member-function mode; `body.Accept(this)` (→ `VisitScriptBlock` builds up to five lambdas); add one sequence point for the whole body if there is none; choose the tiering mode (`_stmtCount > 300`); `CompileTree` each lambda into the optimized or unoptimized fields. Named-block order and names: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitScriptBlock@L2450-L2502. + +### 7.2 Parameter metadata + +Parameters are not compiled. `GetParameterMetaData` builds a `RuntimeDefinedParameterDictionary`: attributes are created by built-in generators (`CmdletBinding`, `Parameter`, `OutputType`, `Alias`, `ValidateSet`, `ValidateNotNull(OrEmpty)`, `DebuggerHidden`, `Experimental`) or a compiled generator per attribute type; positions are assigned automatically (0.. for non-switch parameters) when no explicit positions/parameter sets are used; default values that are constants are stored as values, others as `DefaultValueExpressionWrapper` evaluated lazily with `GetExpressionValue` (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetRuntimeDefinedParameter@L1819-L1907). + +### 7.3 Assignment + +`CompileAssignment` → `ReduceAssignment`: `=` → `IAssignableValue.SetValue`; `+= -= *= /= %=` → dynamic binary operation on `GetValue()` and the right side then `SetValue`; `??=` → `SetValue(Coalesce(get, right))`. Destructuring (`$a, $b = …`) captures the right side as an enumerable and assigns elements through `PSArrayAssignmentRHSBinder`. A converted left side (`[int]$x = …`) stores the attributes with the variable. Variables: slot assignment or `VariableOps.SetVariableValue`. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ReduceAssignment@L993-L1023, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileAssignment@L3527-L3563. + +### 7.4 Tree-walker requirements [INFERENCE] + +A tree-walking evaluator over the AST is a valid start. It must preserve (each item is a protocol or behavior above): the output-pipe model (§6.5) including temporary pipes and `return` writing to the pipe; `$?` rules; per-statement error-action resume (§6.2); loop label matching and the exception form of non-local `break`/`continue`; `switch` testing all clauses; `foreach` enumeration rules; `try/catch` handler matching order, `$_`, `finally` after `return`; variable visibility (slots vs name lookup are an optimization; scopes, `AllScope`, dot-sourcing behavior must not change); error positions (sequence points); evaluation order (right side of an assignment before the left target's index/member evaluation as in `IAssignableValue.GetValue/SetValue`); strict-mode checks on call syntax. See [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). + +## 8. Observable behavior [SPEC] + +Read from source, not run: `1; 2` writes both values; `$x = 1,2` stores an array; `if (gci) {…}` converts the captured output to bool; an error in the 2nd of 3 statements (default `Continue` preference) lets the 3rd run; `switch (1,2) {…}` evaluates `default` per item only when no clause matched; `foreach ($i in $null)` runs zero times for `$null` expression results (enumerable of nothing) — `UNVERIFIED` in detail. Tests: `test/powershell/Language/` (`data/tests.json`). No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Runtime errors raised by the interpreter helpers use `InterpreterError`, whose message IDs are `ParserStrings` keys (so the resx is shared by parse, semantic, compile and runtime errors, see [ERROR-CATALOG.md](ERROR-CATALOG.md) §4). Class-definition errors found by `TypeDefiner` throw `ParseException` at compile time (CON-009). The compile step may throw (e.g. `ScriptHasAdminBlockedContent`, AMSI-detected content → `ScriptContainedMaliciousContent`) and leaves the script block uncompiled. Single-expression evaluation swallows `FlowControlException`. + +## 10. Concurrency and threading + +Compilation of each variant is under `lock(this)` with double-checked flags; the delegates are read without locks after publication. JIT promotion runs on the thread pool ([TIERING-POLICY.md](TIERING-POLICY.md)). `DefaultValueExpressionWrapper` and `ExpressionCache`-style shared state take locks or are immutable. Script block text segments for logging use `Random.Shared`. + +## 11. Extension points and customization hooks + +Attribute generators registered in `Compiler`'s static constructor; `InternalTestHooks` (cache bypass); `ScriptBlock` language-mode capture at creation. Recipes for adding syntax: HOWTO-extend-language (planned T210). + +## 12. Dependencies + +Inbound: all script execution (`ScriptBlock`, `FunctionInfo`, `ExternalScriptInfo`, `PSScriptCmdlet`), constant folding (`IsConstantValueVisitor` calls back into a `Compiler` in `CompilingConstantExpression` mode), `Parser`/`TypeDefiner`. Outbound: the PowerShell binders, `ParserOps`, `PipelineOps`, `ExceptionHandlingOps`, `VariableOps`, `TypeOps`, `System.Linq.Expressions`, the debugger. + +## 13. Performance notes + +Statement dispatch (`switch` + `goto`) was chosen for lists with several statements; the comment warns that JIT time grows with the number of try/catch regions. Optimized variant stores variables in typed tuple fields. Constant folding happens at compile time. + +## 14. Security considerations (mandatory) + +- Before compiling a top-level script block: AMSI scan of the script text (detected content → `ParseException`), checks for admin-blocked content, and the suspicious-content check (hashed pattern table, 29-character maximum pattern) that forces script block logging (`PerformSecurityChecks`, `CheckSuspiciousContent`; AMSI/logging themselves are OUT-OF-SCOPE, see 07-security). +- Language mode is captured when a script block is created and applied on invoke; a switch from a restricted mode to Full Language by dot-sourcing is only audited in Audit mode, otherwise not performed (`InvokeWithPipeImpl`, notes W4). +- `data` statements run in restricted language mode and restore the previous mode in a `finally`. +- Script block logging splits long text into segments of 10,000 plus a random 0..9,999 characters when length ≥ 20,000. + +## 15. Enterprise considerations + +Observability: ETW compile events, script block logging, debugger hooks (`EnterScriptFunction`, `OnSequencePointHit`). Resource limits: none on script size beyond the tiering cutoffs; call-depth limits were not analyzed here (`UNVERIFIED`). Missing: no compile cancellation, no per-script resource accounting. + +## 16. Tests + +`test/powershell/Language/` (flow control, functions, scoping); xUnit tests of the interpreter were not located (VB-014). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| compile to LINQ trees rather than tree-walk | `FACT-SRC` | reuse of the .NET JIT and DLR sites; large compiler class (≈6,600 lines) tightly coupled to the AST | +| per-statement exception wrapper | `FACT-SRC` | script-level continue-on-error semantics | +| output through a mutable pipe object | `FACT-SRC` | `return` and bare expressions behave as output; capturing needs temporary pipes | +| two compile variants (optimized/unoptimized) | `FACT-SRC` | speed vs dynamic variable access (dot-sourcing, debugging) | +| functions compiled lazily when first invoked | `FACT-SRC` | `VisitFunctionDefinition` only defines the function; body compiled on first use | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-ENG-01** [ADAPT] (area: NLD-RUN) Start with a tree-walking evaluator over the AST that implements the protocols of §6 and [IR-SPEC.md](IR-SPEC.md) §5; add an IR later only if profiling demands it. Why: same observable behavior, far less code than a 6,600-line expression-tree compiler. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661. +- **LSN-ENG-02** [REPLICATE] (area: NLD-PIP) Implement statement output as writes to the current output pipe with temporary capture pipes for conditions, assignments, `$()` and `@()`. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureAstResults@L2264-L2355. +- **LSN-ENG-03** [REPLICATE] (area: NLD-ERR) Keep statement-level error handling with resume-at-next-statement, rethrowing flow-control signals untouched. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268. +- **LSN-ENG-04** [REPLICATE] (area: NLD-ERR) Match `try/catch/finally` semantics: typed handlers in order, `$_` bound to an `ErrorRecord`, `finally` runs after `return`, cleanup runs even for a stopped pipeline. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTryStatement@L5247-L5509. +- **LSN-ENG-05** [REPLICATE] (area: NLD-RUN) `switch` evaluates every clause per item; `foreach` wraps a non-enumerable in a one-element sequence; loop labels match case-insensitively with an empty label matching any loop. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitSwitchStatement@L4410-L4542, src/System.Management.Automation/engine/lang/parserutils.cs::LoopFlowException@L36-L67. +- **LSN-ENG-06** [ADAPT] (area: NLD-RUN) Cache parsed script blocks by (file, text) but share only immutable compiled data between clones. Why: PowerShell clears the whole cache at 1024 entries; use an LRU instead. Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.CacheScriptBlock@L609-L637. +- **LSN-ENG-07** [AVOID] (area: NLD-SEC) Do not run an AMSI scan, suspicious-pattern hashing or admin-block checks as part of compilation in MicroteamShell; add a separate policy hook if needed. Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.PerformSecurityChecks@L197-L283. DEV: DEV-ENG-040 + +## 19. Open questions / UNVERIFIED + +- Runtime double evaluation of a side-effecting target in `?.`: the generated tree contains the target expression twice (`GetNullConditionalWrappedExpression`); whether a temp is used was not confirmed (VB-015). +- Behavior of `exit` inside nested script blocks and of `ExitNestedPromptException` (host layer). +- Tests for interpreter/JIT parity (VB-014). + +Verification: `Compiler.cs` was read in 21 windows (notes W1-W21) covering all 63 `Visit*` methods; `CompiledScriptBlock.cs` (2,553 lines) in 9 windows; re-read for this document: `GenerateWhileLoop` (L4760-L4864), `VisitPipeline` (L3736-L3866), compile entry (L2040-L2097); counts (63 Visit methods, 99 helpers) from `data/members-tierA.json` via `tools/gen_T25.py --count`. diff --git a/docs-out/02-language/AST-CATALOG.md b/docs-out/02-language/AST-CATALOG.md new file mode 100644 index 0000000..294bcba --- /dev/null +++ b/docs-out/02-language/AST-CATALOG.md @@ -0,0 +1,269 @@ +--- +id: LANG-ASTCAT-001 +title: AST node catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/ast.cs, src/System.Management.Automation/engine/parser/AstVisitor.cs, src/System.Management.Automation/engine/parser/PreOrderVisitor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/ast-hierarchy.json, data/members-tierA.json, data/visitors.json, data/enums.json] +--- + +# AST node catalog + +> STATIC-ONLY — behavior not run-verified. Every table is `GENERATED` by `docs-out/tools/gen_T23.py` (`APPROX`: regex extraction, see the column notes). Narrative: [02.3-ast.md](02.3-ast.md). + +## 1. Purpose and scope [SPEC] + +Complete list of the AST classes declared in `ast.cs` (76 classes, 11 of them abstract), with base class, visit-order children, parser productions that create them, and the visitors that handle them; the two visitor interface method lists; the inheritance diagram; and the enums declared in `ast.cs`. Count source: `data/ast-hierarchy.json` filtered to `ast.cs` (the dataset has one more row from `VariableAnalysis.cs`, a private helper class, which is excluded). + +## 2. Source map + +| path | type | role | +|---|---|---| +| `src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420` | abstract class | root of all nodes | +| `src/System.Management.Automation/engine/parser/AstVisitor.cs` | file | `ICustomAstVisitor(2)`, `DefaultCustomAstVisitor(2)`, `AstSearcher`, DEBUG checkers | +| `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs` | file | `AstVisitAction`, `AstVisitor(2)`, `IAstPostVisitHandler` | + +## 3. Node catalog + +Column notes: **Base** = direct base class; **Children (visit order)** = receivers of `.InternalVisit(` in the node's `InternalVisit` method, in source order (`—` = leaf or no `InternalVisit`); **v2** = the method mentions `AstVisitor2`, i.e. visiting is gated or extended for second-generation visitors; **Created by** = parser methods containing `new (` (`—` = not created by `Parser.cs`: abstract, compiler-synthesized or created in other files); **Consumed by** = non-default visitor classes that define `Visit` (first six, then `+N`). + +| Node | Kind | Base | Children (visit order, APPROX) | v2 | Created by (Parser.cs) | Consumed by (APPROX) | Source | +|---|---|---|---|---|---|---|---| +| `SequencePointAst` | internal | `Ast` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::SequencePointAst@L424-L451` | +| `ErrorStatementAst` | concrete | `PipelineBaseAst` | `NestedAst[]`, `Flags.Values[].Item2`, `Conditions[]`, `Bodies[]` | | `BlockStatementRule`, `ClassDefinitionRule`, `ConfigurationStatementRule`, `CreateErrorStatementAst`, `DataStatementRule`, `DoWhileStatementRule`, `DynamicKeywordStatementRule`, `EnumDefinitionRule`, `ForStatementRule`, `ForeachStatementRule`, `FunctionDeclarationRule`, `GetKeyValuePair`, `IfStatementRule`, `MethodDeclarationRule`, `ParenthesizedExpressionRule`, `PipelineChainRule`, `StatementRule`, `SwitchStatementRule`, `TrapStatementRule`, `TryStatementRule`, `UsingStatementRule`, `WhileStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +3 | `src/System.Management.Automation/engine/parser/ast.cs::ErrorStatementAst@L456-L648` | +| `ErrorExpressionAst` | concrete | `ExpressionAst` | `NestedAst[]` | | `ArrayLiteralRule`, `AttributeArgumentsRule`, `BinaryExpressionRule`, `DynamicKeywordStatementRule`, `ElementAccessRule`, `ErrorRecoveryParameterInExpression`, `ExpressionRule`, `GetCommandArgument`, `MemberAccessRule`, `RedirectionRule`, `UnaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +3 | `src/System.Management.Automation/engine/parser/ast.cs::ErrorExpressionAst@L653-L707` | +| `ScriptBlockAst` | concrete | `Ast` | `UsingStatements[]`, `Attributes[]`, `ParamBlock`, `DynamicParamBlock`, `BeginBlock`, `ProcessBlock`, `EndBlock`, `CleanBlock` | yes | `NamedBlockListRule`, `ParseFile`, `ScriptBlockBodyRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst@L775-L1739` | +| `ParamBlockAst` | concrete | `Ast` | `Attributes[]`, `Parameters[]` | | `ParamBlockRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ParamBlockAst@L1745-L1858` | +| `NamedBlockAst` | concrete | `Ast` | `Traps, Statements (static helper)` | | `NamedBlockListRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::NamedBlockAst@L1864-L2026` | +| `NamedAttributeArgumentAst` | concrete | `Ast` | `Argument` | | `AttributeArgumentsRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::NamedAttributeArgumentAst@L2032-L2113` | +| `AttributeBaseAst` | abstract | `Ast` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::AttributeBaseAst@L2119-L2146` | +| `AttributeAst` | concrete | `AttributeBaseAst` | `PositionalArguments[]`, `NamedArguments[]` | | `AttributeRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `FindVariablesVisitor`, `GetSafeValueVisitor` +8 | `src/System.Management.Automation/engine/parser/ast.cs::AttributeAst@L2151-L2257` | +| `TypeConstraintAst` | concrete | `AttributeBaseAst` | — | | `AttributeRule`, `ClassDefinitionRule`, `EnumDefinitionRule` | `AstSearcher`, `CheckAllParentsSet`, `CheckTypeBuilder`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor` +7 | `src/System.Management.Automation/engine/parser/ast.cs::TypeConstraintAst@L2262-L2317` | +| `ParameterAst` | concrete | `Ast` | `Attributes[]`, `Name`, `DefaultValue` | | `ParameterRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `FindVariablesVisitor`, `GetSafeValueVisitor` +8 | `src/System.Management.Automation/engine/parser/ast.cs::ParameterAst@L2323-L2512` | +| `StatementBlockAst` | concrete | `Ast` | `traps[]`, `statements[]` | | `NamedBlockListRule`, `ParseFile`, `ScriptBlockBodyRule`, `StatementBlockRule`, `SubExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::StatementBlockAst@L2522-L2624` | +| `StatementAst` | abstract | `Ast` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::StatementAst@L2629-L2642` | +| `TypeDefinitionAst` | concrete | `StatementAst` | `Attributes[]`, `BaseTypes[]`, `Members[]` | yes | `ClassDefinitionRule`, `EnumDefinitionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +8 | `src/System.Management.Automation/engine/parser/ast.cs::TypeDefinitionAst@L2667-L2865` | +| `UsingStatementAst` | concrete | `StatementAst` | `Name`, `ModuleSpecification`, `Alias` | yes | `UsingStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::UsingStatementAst@L2901-L3105` | +| `MemberAst` | abstract | `Ast` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::MemberAst@L3110-L3126` | +| `PropertyMemberAst` | concrete | `MemberAst` | `PropertyType`, `Attributes[]`, `InitialValue` | yes | `ClassMemberRule`, `EnumMemberRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::PropertyMemberAst@L3156-L3315` | +| `FunctionMemberAst` | concrete | `MemberAst` | `Attributes[]`, `ReturnType`, `_functionDefinitionAst` | yes | `ClassMemberRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +6 | `src/System.Management.Automation/engine/parser/ast.cs::FunctionMemberAst@L3342-L3633` | +| `CompilerGeneratedMemberFunctionAst` | internal | `MemberAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::CompilerGeneratedMemberFunctionAst@L3642-L3740` | +| `FunctionDefinitionAst` | concrete | `StatementAst` | `Parameters[]`, `Body` | | `FunctionDeclarationRule`, `MethodDeclarationRule` | `AstSearcher`, `CheckAllParentsSet`, `CheckBreakpointInScript`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor` +15 | `src/System.Management.Automation/engine/parser/ast.cs::FunctionDefinitionAst@L3745-L4054` | +| `IfStatementAst` | concrete | `StatementAst` | `Clauses[].Item1`, `Clauses[].Item2`, `ElseClause` | | `IfStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::IfStatementAst@L4059-L4157` | +| `DataStatementAst` | concrete | `StatementAst` | `Body` | | `DataStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `FindAllVariablesVisitor` +9 | `src/System.Management.Automation/engine/parser/ast.cs::DataStatementAst@L4162-L4259` | +| `LabeledStatementAst` | abstract | `StatementAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::LabeledStatementAst@L4266-L4303` | +| `LoopStatementAst` | abstract | `LabeledStatementAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::LoopStatementAst@L4309-L4337` | +| `ForEachStatementAst` | concrete | `LoopStatementAst` | `Variable`, `Condition`, `Body` | | `ForeachStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `FindAllVariablesVisitor` +9 | `src/System.Management.Automation/engine/parser/ast.cs::ForEachStatementAst@L4363-L4484` | +| `ForStatementAst` | concrete | `LoopStatementAst` | `Initializer`, `Condition`, `Iterator`, `Body` | | `ForStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ForStatementAst@L4489-L4571` | +| `DoWhileStatementAst` | concrete | `LoopStatementAst` | `Condition`, `Body` | | `DoWhileStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::DoWhileStatementAst@L4576-L4627` | +| `DoUntilStatementAst` | concrete | `LoopStatementAst` | `Condition`, `Body` | | `DoWhileStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::DoUntilStatementAst@L4632-L4683` | +| `WhileStatementAst` | concrete | `LoopStatementAst` | `Condition`, `Body` | | `WhileStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::WhileStatementAst@L4688-L4739` | +| `SwitchStatementAst` | concrete | `LabeledStatementAst` | `Condition`, `Clauses[].Item1`, `Clauses[].Item2`, `Default` | | `SwitchStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `FindAllVariablesVisitor` +7 | `src/System.Management.Automation/engine/parser/ast.cs::SwitchStatementAst@L4791-L4913` | +| `CatchClauseAst` | concrete | `Ast` | `CatchTypes[]`, `Body` | | `CatchBlockRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::CatchClauseAst@L4922-L5010` | +| `TryStatementAst` | concrete | `StatementAst` | `Body`, `CatchClauses[]`, `Finally` | | `TryStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::TryStatementAst@L5015-L5130` | +| `TrapStatementAst` | concrete | `StatementAst` | `TrapType`, `Body` | | `TrapStatementRule` | — | `src/System.Management.Automation/engine/parser/ast.cs::TrapStatementAst@L5135-L5206` | +| `BreakStatementAst` | concrete | `StatementAst` | `Label` | | `BreakStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::BreakStatementAst@L5215-L5267` | +| `ContinueStatementAst` | concrete | `StatementAst` | `Label` | | `ContinueStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ContinueStatementAst@L5272-L5324` | +| `ReturnStatementAst` | concrete | `StatementAst` | `Pipeline` | | `ReturnStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ReturnStatementAst@L5329-L5381` | +| `ExitStatementAst` | concrete | `StatementAst` | `Pipeline` | | `ExitStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::ExitStatementAst@L5386-L5438` | +| `ThrowStatementAst` | concrete | `StatementAst` | `Pipeline` | | `ThrowStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::ThrowStatementAst@L5443-L5529` | +| `ChainableAst` | abstract | `PipelineBaseAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::ChainableAst@L5534-L5543` | +| `PipelineChainAst` | concrete | `ChainableAst` | `LhsPipelineChain`, `RhsPipeline` | yes | `PipelineChainRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `SafeExprEvaluator` | `src/System.Management.Automation/engine/parser/ast.cs::PipelineChainAst@L5549-L5647` | +| `PipelineBaseAst` | abstract | `StatementAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::PipelineBaseAst@L5657-L5678` | +| `PipelineAst` | concrete | `ChainableAst` | `PipelineElements[]` | | `PipelineRule`, `SwitchStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +6 | `src/System.Management.Automation/engine/parser/ast.cs::PipelineAst@L5684-L5825` | +| `CommandElementAst` | abstract | `Ast` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::CommandElementAst@L5830-L5843` | +| `CommandParameterAst` | concrete | `CommandElementAst` | `Argument` | | `CommandRule`, `ErrorRecoveryParameterInExpression` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::CommandParameterAst@L5853-L5944` | +| `CommandBaseAst` | abstract | `StatementAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::CommandBaseAst@L5949-L5982` | +| `CommandAst` | concrete | `CommandBaseAst` | `CommandElements[]`, `Redirections[]` | | `CommandRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `FindAllVariablesVisitor` +10 | `src/System.Management.Automation/engine/parser/ast.cs::CommandAst@L5987-L6111` | +| `CommandExpressionAst` | concrete | `CommandBaseAst` | `Expression`, `Redirections[]` | | `MethodDeclarationRule`, `PipelineRule`, `SwitchStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::CommandExpressionAst@L6116-L6186` | +| `RedirectionAst` | abstract | `Ast` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::RedirectionAst@L6191-L6208` | +| `MergingRedirectionAst` | concrete | `RedirectionAst` | — | | `RedirectionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::MergingRedirectionAst@L6255-L6297` | +| `FileRedirectionAst` | concrete | `RedirectionAst` | `Location` | | `RedirectionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::FileRedirectionAst@L6302-L6372` | +| `AssignmentStatementAst` | concrete | `PipelineBaseAst` | `Left`, `Right` | | `PipelineChainRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `FindVariablesVisitor` +9 | `src/System.Management.Automation/engine/parser/ast.cs::AssignmentStatementAst@L6379-L6495` | +| `ConfigurationDefinitionAst` | concrete | `StatementAst` | `InstanceName`, `Body` | yes | `ConfigurationStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::ConfigurationDefinitionAst@L6516-L6998` | +| `DynamicKeywordStatementAst` | concrete | `StatementAst` | `CommandElements[]` | yes | `CommandRule`, `DynamicKeywordStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::DynamicKeywordStatementAst@L7003-L7335` | +| `ExpressionAst` | abstract | `CommandElementAst` | — | | — | — | `src/System.Management.Automation/engine/parser/ast.cs::ExpressionAst@L7344-L7399` | +| `TernaryExpressionAst` | concrete | `ExpressionAst` | `Condition`, `IfTrue`, `IfFalse` | yes | `ExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::TernaryExpressionAst@L7404-L7497` | +| `BinaryExpressionAst` | concrete | `ExpressionAst` | `Left`, `Right` | | `BinaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::BinaryExpressionAst@L7502-L7617` | +| `UnaryExpressionAst` | concrete | `ExpressionAst` | `Child` | | `UnaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::UnaryExpressionAst@L7622-L7704` | +| `BlockStatementAst` | concrete | `StatementAst` | `Body` | | `BlockStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +6 | `src/System.Management.Automation/engine/parser/ast.cs::BlockStatementAst@L7710-L7773` | +| `AttributedExpressionAst` | concrete | `ExpressionAst` | `Attribute`, `Child` | | `UnaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::AttributedExpressionAst@L7779-L7901` | +| `ConvertExpressionAst` | concrete | `AttributedExpressionAst` | `Type`, `Child` | | `UnaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +6 | `src/System.Management.Automation/engine/parser/ast.cs::ConvertExpressionAst@L7906-L7990` | +| `MemberExpressionAst` | concrete | `ExpressionAst` | `Expression`, `Member` | | `DynamicKeywordStatementRule`, `MemberAccessRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::MemberExpressionAst@L7997-L8110` | +| `InvokeMemberExpressionAst` | concrete | `MemberExpressionAst` | `Expression`, `Member`, `Arguments[]` | | `MemberInvokeRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `ExportVisitor`, `GetSafeValueVisitor` +6 | `src/System.Management.Automation/engine/parser/ast.cs::InvokeMemberExpressionAst@L8115-L8310` | +| `BaseCtorInvokeMemberExpressionAst` | concrete | `InvokeMemberExpressionAst` | — | yes | `MethodDeclarationRule` | `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor`, `IsSafeValueVisitor`, `SafeExprEvaluator` +1 | `src/System.Management.Automation/engine/parser/ast.cs::BaseCtorInvokeMemberExpressionAst@L8315-L8363` | +| `TypeExpressionAst` | concrete | `ExpressionAst` | — | | `UnaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +6 | `src/System.Management.Automation/engine/parser/ast.cs::TypeExpressionAst@L9345-L9398` | +| `VariableExpressionAst` | concrete | `ExpressionAst` | — | | `CheckUsingVariable`, `ForeachStatementRule`, `ParameterRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `FindAllVariablesVisitor`, `GetSafeValueVisitor` +7 | `src/System.Management.Automation/engine/parser/ast.cs::VariableExpressionAst@L9404-L9624` | +| `ConstantExpressionAst` | concrete | `ExpressionAst` | — | | `AttributeArgumentsRule`, `PrimaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::ConstantExpressionAst@L9631-L9688` | +| `StringConstantExpressionAst` | concrete | `ConstantExpressionAst` | — | | `CommandRule`, `DynamicKeywordStatementRule`, `ExpandableStringRule`, `GetCommandArgument`, `InlineScriptRule`, `ParseNestedExpressions`, `PrimaryExpressionRule`, `ResolveUsingAssembly`, `SimpleNameRule`, `SwitchStatementRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::StringConstantExpressionAst@L9732-L9819` | +| `ExpandableStringExpressionAst` | concrete | `ExpressionAst` | `NestedExpressions[]` | | `ExpandableStringRule`, `GetCommandArgument` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ExpandableStringExpressionAst@L9825-L9974` | +| `ScriptBlockExpressionAst` | concrete | `ExpressionAst` | `ScriptBlock` | | `ScriptBlockExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `CheckBreakpointInScript`, `Compiler`, `ConstantValueVisitor`, `FindAllVariablesVisitor` +13 | `src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockExpressionAst@L9979-L10041` | +| `ArrayLiteralAst` | concrete | `ExpressionAst` | `Elements[]` | | `ArrayLiteralRule`, `GetCommandArgument`, `UnaryExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::ArrayLiteralAst@L10048-L10123` | +| `HashtableAst` | concrete | `ExpressionAst` | `KeyValuePairs[].Item1`, `KeyValuePairs[].Item2` | | `HashExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +5 | `src/System.Management.Automation/engine/parser/ast.cs::HashtableAst@L10128-L10218` | +| `ArrayExpressionAst` | concrete | `ExpressionAst` | `SubExpression` | | `SubExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::ArrayExpressionAst@L10224-L10284` | +| `ParenExpressionAst` | concrete | `ExpressionAst` | `Pipeline` | | `ParenthesizedExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::ParenExpressionAst@L10290-L10350` | +| `SubExpressionAst` | concrete | `ExpressionAst` | `SubExpression` | | `SubExpressionRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::SubExpressionAst@L10356-L10411` | +| `UsingExpressionAst` | concrete | `ExpressionAst` | `SubExpression` | | `CheckUsingVariable` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `FindAllVariablesVisitor`, `GetSafeValueVisitor` +7 | `src/System.Management.Automation/engine/parser/ast.cs::UsingExpressionAst@L10416-L10550` | +| `IndexExpressionAst` | concrete | `ExpressionAst` | `Target`, `Index` | | `ElementAccessRule` | `AstSearcher`, `CheckAllParentsSet`, `Compiler`, `ConstantValueVisitor`, `GetSafeValueVisitor`, `IsConstantValueVisitor` +4 | `src/System.Management.Automation/engine/parser/ast.cs::IndexExpressionAst@L10555-L10646` | + +Rows: **76** (`GENERATED`). Concrete nodes with no recognized child visit: 9 (leaves, or children visited through a different construct). + +## 4. Inheritance diagram + +```mermaid +flowchart LR + Ast --> SequencePointAst + PipelineBaseAst --> ErrorStatementAst + ExpressionAst --> ErrorExpressionAst + Ast --> ScriptBlockAst + Ast --> ParamBlockAst + Ast --> NamedBlockAst + Ast --> NamedAttributeArgumentAst + Ast --> AttributeBaseAst + AttributeBaseAst --> AttributeAst + AttributeBaseAst --> TypeConstraintAst + Ast --> ParameterAst + Ast --> StatementBlockAst + Ast --> StatementAst + StatementAst --> TypeDefinitionAst + StatementAst --> UsingStatementAst + Ast --> MemberAst + MemberAst --> PropertyMemberAst + MemberAst --> FunctionMemberAst + MemberAst --> CompilerGeneratedMemberFunctionAst + StatementAst --> FunctionDefinitionAst + StatementAst --> IfStatementAst + StatementAst --> DataStatementAst + StatementAst --> LabeledStatementAst + LabeledStatementAst --> LoopStatementAst + LoopStatementAst --> ForEachStatementAst + LoopStatementAst --> ForStatementAst + LoopStatementAst --> DoWhileStatementAst + LoopStatementAst --> DoUntilStatementAst + LoopStatementAst --> WhileStatementAst + LabeledStatementAst --> SwitchStatementAst + Ast --> CatchClauseAst + StatementAst --> TryStatementAst + StatementAst --> TrapStatementAst + StatementAst --> BreakStatementAst + StatementAst --> ContinueStatementAst + StatementAst --> ReturnStatementAst + StatementAst --> ExitStatementAst + StatementAst --> ThrowStatementAst + PipelineBaseAst --> ChainableAst + ChainableAst --> PipelineChainAst + StatementAst --> PipelineBaseAst + ChainableAst --> PipelineAst + Ast --> CommandElementAst + CommandElementAst --> CommandParameterAst + StatementAst --> CommandBaseAst + CommandBaseAst --> CommandAst + CommandBaseAst --> CommandExpressionAst + Ast --> RedirectionAst + RedirectionAst --> MergingRedirectionAst + RedirectionAst --> FileRedirectionAst + PipelineBaseAst --> AssignmentStatementAst + StatementAst --> ConfigurationDefinitionAst + StatementAst --> DynamicKeywordStatementAst + CommandElementAst --> ExpressionAst + ExpressionAst --> TernaryExpressionAst + ExpressionAst --> BinaryExpressionAst + ExpressionAst --> UnaryExpressionAst + StatementAst --> BlockStatementAst + ExpressionAst --> AttributedExpressionAst + AttributedExpressionAst --> ConvertExpressionAst + ExpressionAst --> MemberExpressionAst + MemberExpressionAst --> InvokeMemberExpressionAst + InvokeMemberExpressionAst --> BaseCtorInvokeMemberExpressionAst + ExpressionAst --> TypeExpressionAst + ExpressionAst --> VariableExpressionAst + ExpressionAst --> ConstantExpressionAst + ConstantExpressionAst --> StringConstantExpressionAst + ExpressionAst --> ExpandableStringExpressionAst + ExpressionAst --> ScriptBlockExpressionAst + ExpressionAst --> ArrayLiteralAst + ExpressionAst --> HashtableAst + ExpressionAst --> ArrayExpressionAst + ExpressionAst --> ParenExpressionAst + ExpressionAst --> SubExpressionAst + ExpressionAst --> UsingExpressionAst + ExpressionAst --> IndexExpressionAst +``` + +## 5. Visitor interfaces + +Method lists are the `Visit*` methods found in each type (`GENERATED`, `APPROX`). `ICustomAstVisitor` returns `object`; `AstVisitor` returns `AstVisitAction`. + +**ICustomAstVisitor** (`src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor@L14-L202`, 54 visit methods): `VisitArrayExpression`, `VisitArrayLiteral`, `VisitAssignmentStatement`, `VisitAttribute`, `VisitAttributedExpression`, `VisitBinaryExpression`, `VisitBlockStatement`, `VisitBreakStatement`, `VisitCatchClause`, `VisitCommand`, `VisitCommandExpression`, `VisitCommandParameter`, `VisitConstantExpression`, `VisitContinueStatement`, `VisitConvertExpression`, `VisitDataStatement`, `VisitDoUntilStatement`, `VisitDoWhileStatement`, `VisitErrorExpression`, `VisitErrorStatement`, `VisitExitStatement`, `VisitExpandableStringExpression`, `VisitFileRedirection`, `VisitForEachStatement`, `VisitForStatement`, `VisitFunctionDefinition`, `VisitHashtable`, `VisitIfStatement`, `VisitIndexExpression`, `VisitInvokeMemberExpression`, `VisitMemberExpression`, `VisitMergingRedirection`, `VisitNamedAttributeArgument`, `VisitNamedBlock`, `VisitParamBlock`, `VisitParameter`, `VisitParenExpression`, `VisitPipeline`, `VisitReturnStatement`, `VisitScriptBlock`, `VisitScriptBlockExpression`, `VisitStatementBlock`, `VisitStringConstantExpression`, `VisitSubExpression`, `VisitSwitchStatement`, `VisitThrowStatement`, `VisitTrap`, `VisitTryStatement`, `VisitTypeConstraint`, `VisitTypeExpression`, `VisitUnaryExpression`, `VisitUsingExpression`, `VisitVariableExpression`, `VisitWhileStatement` + +**ICustomAstVisitor2** (`src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235`, 9 visit methods): `VisitBaseCtorInvokeMemberExpression`, `VisitConfigurationDefinition`, `VisitDynamicKeywordStatement`, `VisitFunctionMember`, `VisitPipelineChain`, `VisitPropertyMember`, `VisitTernaryExpression`, `VisitTypeDefinition`, `VisitUsingStatement` + +**AstVisitor** (`src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215`, 54 visit methods): `VisitArrayExpression`, `VisitArrayLiteral`, `VisitAssignmentStatement`, `VisitAttribute`, `VisitAttributedExpression`, `VisitBinaryExpression`, `VisitBlockStatement`, `VisitBreakStatement`, `VisitCatchClause`, `VisitCommand`, `VisitCommandExpression`, `VisitCommandParameter`, `VisitConstantExpression`, `VisitContinueStatement`, `VisitConvertExpression`, `VisitDataStatement`, `VisitDoUntilStatement`, `VisitDoWhileStatement`, `VisitErrorExpression`, `VisitErrorStatement`, `VisitExitStatement`, `VisitExpandableStringExpression`, `VisitFileRedirection`, `VisitForEachStatement`, `VisitForStatement`, `VisitFunctionDefinition`, `VisitHashtable`, `VisitIfStatement`, `VisitIndexExpression`, `VisitInvokeMemberExpression`, `VisitMemberExpression`, `VisitMergingRedirection`, `VisitNamedAttributeArgument`, `VisitNamedBlock`, `VisitParamBlock`, `VisitParameter`, `VisitParenExpression`, `VisitPipeline`, `VisitReturnStatement`, `VisitScriptBlock`, `VisitScriptBlockExpression`, `VisitStatementBlock`, `VisitStringConstantExpression`, `VisitSubExpression`, `VisitSwitchStatement`, `VisitThrowStatement`, `VisitTrap`, `VisitTryStatement`, `VisitTypeConstraint`, `VisitTypeExpression`, `VisitUnaryExpression`, `VisitUsingExpression`, `VisitVariableExpression`, `VisitWhileStatement` + +**AstVisitor2** (`src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor2@L220-L248`, 9 visit methods): `VisitBaseCtorInvokeMemberExpression`, `VisitConfigurationDefinition`, `VisitDynamicKeywordStatement`, `VisitFunctionMember`, `VisitPipelineChain`, `VisitPropertyMember`, `VisitTernaryExpression`, `VisitTypeDefinition`, `VisitUsingStatement` + +## 6. Visitor implementations in the repository + +| Class | Bases | Visit methods defined | File | +|---|---|---:|---| +| `ScriptBlockTelemetry` | `AstVisitor2` | 10 | `src/System.Management.Automation/utils/PSTelemetryMethods.cs` | +| `MyAstVisitor2` | `AstVisitor2` | 0 | `test/xUnit/csharp/test_AstDefaultVisit.cs` | +| `MyDefaultCustomAstVisitor2` | `DefaultCustomAstVisitor2` | 0 | `test/xUnit/csharp/test_AstDefaultVisit.cs` | +| `MyICustomAstVisitor2` | `ICustomAstVisitor2` | 0 | `test/xUnit/csharp/test_AstDefaultVisit.cs` | +| `FindFunctionsVisitor` | `AstVisitor` | 1 | `src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs` | +| `FindVariablesVisitor` | `AstVisitor2` | 8 | `src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs` | +| `ExportVisitor` | `AstVisitor2` | 14 | `src/System.Management.Automation/engine/Modules/ScriptAnalysis.cs` | +| `AstSearcher` | `AstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `AstVisitor` | — | 54 | `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs` | +| `AstVisitor2` | `AstVisitor` | 9 | `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs` | +| `AstVisitor2` | `AstVisitor` | 9 | `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs` | +| `CheckAllParentsSet` | `AstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `CheckTypeBuilder` | `AstVisitor2` | 1 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `Compiler` | `ICustomAstVisitor2` | 63 | `src/System.Management.Automation/engine/parser/Compiler.cs` | +| `ConstantValueVisitor` | `ICustomAstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/ConstantValues.cs` | +| `DefaultCustomAstVisitor` | `ICustomAstVisitor` | 54 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `DefaultCustomAstVisitor2` | `DefaultCustomAstVisitor`, `ICustomAstVisitor2` | 9 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `FindAllVariablesVisitor` | `AstVisitor` | 11 | `src/System.Management.Automation/engine/parser/VariableAnalysis.cs` | +| `GetSafeValueVisitor` | `ICustomAstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/SafeValues.cs` | +| `ICustomAstVisitor` | — | 54 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `ICustomAstVisitor2` | `ICustomAstVisitor` | 9 | `src/System.Management.Automation/engine/parser/AstVisitor.cs` | +| `IsConstantValueVisitor` | `ICustomAstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/ConstantValues.cs` | +| `IsSafeValueVisitor` | `ICustomAstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/SafeValues.cs` | +| `RestrictedLanguageChecker` | `AstVisitor` | 52 | `src/System.Management.Automation/engine/parser/SemanticChecks.cs` | +| `SemanticChecks` | `AstVisitor2`, `IAstPostVisitHandler` | 32 | `src/System.Management.Automation/engine/parser/SemanticChecks.cs` | +| `SymbolResolvePostActionVisitor` | `DefaultCustomAstVisitor2` | 4 | `src/System.Management.Automation/engine/parser/SymbolResolver.cs` | +| `SymbolResolver` | `AstVisitor2`, `IAstPostVisitHandler` | 13 | `src/System.Management.Automation/engine/parser/SymbolResolver.cs` | +| `VariableAnalysis` | `ICustomAstVisitor2` | 62 | `src/System.Management.Automation/engine/parser/VariableAnalysis.cs` | +| `CheckBreakpointInScript` | `AstVisitor` | 2 | `src/System.Management.Automation/engine/debugger/Breakpoint.cs` | +| `SafeExprEvaluator` | `ICustomAstVisitor2` | 63 | `src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs` | +| `ScriptBlockToPowerShellChecker` | `AstVisitor` | 8 | `src/System.Management.Automation/engine/runtime/ScriptBlockToPowerShell.cs` | +| `TypeInferenceVisitor` | `ICustomAstVisitor2` | 0 | `src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs` | +| `VariableAssignmentVisitor` | `AstVisitor2` | 9 | `src/System.Management.Automation/engine/parser/TypeInferenceVisitor.cs` | +| `UsingExpressionAstSearcher` | `AstSearcher` | 1 | `src/System.Management.Automation/engine/runtime/ScriptBlockToPowerShell.cs` | + +## 7. Enums declared in ast.cs + +- `TypeAttributes` (`src/System.Management.Automation/engine/parser/ast.cs::TypeAttributes@L2648-L2661`): `None`=0x00, `Class`=0x01, `Interface`=0x02, `Enum`=0x04 +- `UsingStatementKind` (`src/System.Management.Automation/engine/parser/ast.cs::UsingStatementKind@L2870-L2896`): `Assembly`=0, `Command`=1, `Module`=2, `Namespace`=3, `Type`=4 +- `PropertyAttributes` (`src/System.Management.Automation/engine/parser/ast.cs::PropertyAttributes@L3132-L3151`): `None`=0x00, `Public`=0x01, `Private`=0x02, `Static`=0x10, `Literal`=0x20, `Hidden`=0x40 +- `MethodAttributes` (`src/System.Management.Automation/engine/parser/ast.cs::MethodAttributes@L3321-L3337`): `None`=0x00, `Public`=0x01, `Private`=0x02, `Static`=0x10, `Hidden`=0x40 +- `SpecialMemberFunctionType` (`src/System.Management.Automation/engine/parser/ast.cs::SpecialMemberFunctionType@L3635-L3640`): `None`, `DefaultConstructor`, `StaticConstructor` +- `ForEachFlags` (`src/System.Management.Automation/engine/parser/ast.cs::ForEachFlags@L4344-L4358`): `None`=0x00, `Parallel`=0x01 +- `SwitchFlags` (`src/System.Management.Automation/engine/parser/ast.cs::SwitchFlags@L4746-L4786`): `None`=0x00, `File`=0x01, `Regex`=0x02, `Wildcard`=0x04, `Exact`=0x08, `CaseSensitive`=0x10, `Parallel`=0x20 +- `RedirectionStream` (`src/System.Management.Automation/engine/parser/ast.cs::RedirectionStream@L6214-L6250`): `All`=0, `Output`=1, `Error`=2, `Warning`=3, `Verbose`=4, `Debug`=5, `Information`=6 +- `ConfigurationType` (`src/System.Management.Automation/engine/parser/ast.cs::ConfigurationType@L6500-L6511`): `Resource`=0, `Meta`=1 +- `StringConstantType` (`src/System.Management.Automation/engine/parser/ast.cs::StringConstantType@L9693-L9725`): `SingleQuoted`, `SingleQuotedHereString`, `DoubleQuoted`, `DoubleQuotedHereString`, `BareWord` + +## 8. Non-AST public types in ast.cs + +Type names (`ITypeName` family), `ScriptRequirements` and `CommentHelpInfo` live in `ast.cs` but are not `Ast` nodes (read in notes, `FACT-SRC`): see [02.3-ast.md](02.3-ast.md) §5. + +Verification: node rows reconciled with `data/ast-hierarchy.json` (filtered by file); `InternalVisit` ranges come from `data/members-tierA.json`; edges in the diagram are the direct base of each row; visitor lists reconcile with `data/visitors.json`. + diff --git a/docs-out/02-language/COMPILATION-MAP.md b/docs-out/02-language/COMPILATION-MAP.md new file mode 100644 index 0000000..502169d --- /dev/null +++ b/docs-out/02-language/COMPILATION-MAP.md @@ -0,0 +1,206 @@ +--- +id: LANG-COMPMAP-001 +title: Compilation map +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Compiler.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/members-tierA.json] +--- + +# Compilation map + +> STATIC-ONLY — behavior not run-verified. Rows are `GENERATED` by `docs-out/tools/gen_T25.py`: method names and line ranges from `data/members-tierA.json` (`APPROX`), the "compiled form" text from the hand-written annotation file `docs-out/tools/annotations/T25-compilation-map.json` (`FACT-SRC`, read in `Compiler.cs`). Narrative: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md). Intermediate representation: [IR-SPEC.md](IR-SPEC.md). + +## 1. Purpose and scope [SPEC] + +For every AST node type, how `Compiler` (an `ICustomAstVisitor2`) turns it into a LINQ expression tree. `Compiler` has 63 `Visit*` methods (one per visitor method of `ICustomAstVisitor` plus `ICustomAstVisitor2`, see [AST-CATALOG.md](AST-CATALOG.md) §5) and 99 other methods (section 4). + +## 2. Source map + +| path | type | role | +|---|---|---| +| `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L818-L6658` | class | AST → expression-tree compiler | + +## 3. Node → compiled form + +| AST node (Visit method) | Compiled form | How (FACT-SRC) | Source | +|---|---|---|---| +| `ErrorStatementAst` (`VisitErrorStatement`) | not compiled | returns null; error nodes never reach the compiler because a script with parse errors cannot be turned into a script block | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitErrorStatement@L2438-L2441` | +| `ErrorExpressionAst` (`VisitErrorExpression`) | constant | returns Constant(1); placeholder | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitErrorExpression@L2443-L2446` | +| `ScriptBlockAst` (`VisitScriptBlock`) | five lambdas | builds one Expression> per named block (DynamicParam, Begin, Process, End, Clean); function name suffixes `` etc.; param block is not compiled (defaults come from parameter metadata); the first compiled block also defines classes/usings when the script block is the root | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitScriptBlock@L2450-L2502` | +| `TypeConstraintAst` (`VisitTypeConstraint`) | not visited | assert false; type constraints are handled by their parents | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTypeConstraint@L2995-L2999` | +| `AttributeAst` (`VisitAttribute`) | not visited | assert false; attributes are turned into attribute objects by GetAttribute at metadata time | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitAttribute@L3001-L3005` | +| `NamedAttributeArgumentAst` (`VisitNamedAttributeArgument`) | not visited | assert false | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitNamedAttributeArgument@L3007-L3011` | +| `ParameterAst` (`VisitParameter`) | not visited | assert false; parameters become RuntimeDefinedParameter metadata (GetParameterMetaData) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitParameter@L3013-L3017` | +| `ParamBlockAst` (`VisitParamBlock`) | not visited | assert false | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitParamBlock@L3019-L3023` | +| `NamedBlockAst` (`VisitNamedBlock`) | not visited | assert false; CompileNamedBlock is called from VisitScriptBlock | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitNamedBlock@L3025-L3029` | +| `StatementBlockAst` (`VisitStatementBlock`) | statement list | CompileStatementListWithTraps -> Block (Empty if no statements); each statement wrapped for error-action handling | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitStatementBlock@L3031-L3042` | +| `TypeDefinitionAst` (`VisitTypeDefinition`) | empty | Empty; types are defined by GenerateTypesAndUsings for the root block | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTypeDefinition@L3425-L3428` | +| `PropertyMemberAst` (`VisitPropertyMember`) | not compiled | returns null | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitPropertyMember@L3430-L3433` | +| `FunctionMemberAst` (`VisitFunctionMember`) | not compiled | returns null; member bodies are compiled as separate script blocks with member-function mode | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitFunctionMember@L3435-L3438` | +| `BaseCtorInvokeMemberExpressionAst` (`VisitBaseCtorInvokeMemberExpression`) | call | InvokeBaseCtorMethod: dynamic site with PSInvokeBaseCtorBinder | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitBaseCtorInvokeMemberExpression@L3440-L3446` | +| `UsingStatementAst` (`VisitUsingStatement`) | empty | Empty; usings are loaded by GenerateLoadUsings in the root prolog | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitUsingStatement@L3448-L3451` | +| `ConfigurationDefinitionAst` (`VisitConfigurationDefinition`) | synthesized pipeline | compiles the AST built by GenerateSetItemPipelineAst (set-item function:); DSC, out of scope | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitConfigurationDefinition@L3453-L3456` | +| `DynamicKeywordStatementAst` (`VisitDynamicKeywordStatement`) | synthesized pipeline | meta statements -> Empty; otherwise compiles the AST built by GenerateCommandCallPipelineAst; DSC, out of scope | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitDynamicKeywordStatement@L3458-L3466` | +| `FunctionDefinitionAst` (`VisitFunctionDefinition`) | runtime call | FunctionOps.DefineFunction(context, ast, ScriptBlockExpressionWrapper); the function body is not compiled with the parent, only when the wrapper's script block first runs | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitFunctionDefinition@L3468-L3475` | +| `IfStatementAst` (`VisitIfStatement`) | conditional | nested Expression.IfThenElse from last clause to first; condition = CaptureStatementResults(cond, Condition) converted to bool; position update before each condition | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitIfStatement@L3477-L3514` | +| `TrapAst` (`VisitTrap`) | not visited | assert false; traps are compiled by CompileTrap from statement lists | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTrap@L3516-L3520` | +| `AssignmentStatementAst` (`VisitAssignmentStatement`) | assignment | CompileAssignment: destructuring (array LHS) uses PSArrayAssignmentRHSBinder; otherwise ReduceAssignment through IAssignableValue (SetValue / dynamic binary op for +=, -=, ...) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitAssignmentStatement@L3522-L3525` | +| `PipelineChainAst` (`VisitPipelineChain`) | dispatch loop | && / \|\| over pipelines: statement-index dispatch with try/catch (like statement lists), `if ($?)` / `if (-not $?)` between elements, background chains call PipelineOps.InvokePipelineInBackground; no trap handlers are pushed | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitPipelineChain@L3565-L3717` | +| `PipelineAst` (`VisitPipeline`) | runtime call | single expression -> compiled expression (with redirections via GetRedirectedExpression); otherwise PipelineOps.InvokePipeline(input, ignoreInput, CommandParameterInternal[][] per command, command asts, redirections, funcContext); background -> InvokePipelineInBackground; position update unless parent is assignment/paren | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitPipeline@L3736-L3866` | +| `MergingRedirectionAst` (`VisitMergingRedirection`) | runtime object | returns a MergingRedirection value (not an Expression) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitMergingRedirection@L4140-L4147` | +| `FileRedirectionAst` (`VisitFileRedirection`) | new object | Expression.New(FileRedirection(stream, append, fileName)); non-constant file names converted with PSToStringBinder | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitFileRedirection@L4149-L4175` | +| `CommandAst` (`VisitCommand`) | array of arguments | NewArrayInit(CommandParameterInternal[]): first element is the command name argument, then parameters/arguments; numeric literals keep their source text (WrappedNumber); strict-mode check for `f (args)` call syntax | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitCommand@L4177-L4237` | +| `CommandExpressionAst` (`VisitCommandExpression`) | expression + output | compile expression; unless pre/post increment or void, write value to funcContext._outputPipe (CallAddPipe) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitCommandExpression@L4268-L4279` | +| `CommandParameterAst` (`VisitCommandParameter`) | argument object | CommandParameterInternal.CreateParameter / CreateParameterWithArgument | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitCommandParameter@L4281-L4305` | +| `SwitchStatementAst` (`VisitSwitchStatement`) | loop | saves/restores `$_`; -file reads lines with StreamReader; otherwise GenerateIteratorStatement with `$switch`; per item every clause is tested (constant script block, regex, wildcard, exact string, or PSSwitchClauseEvalBinder) - all matching clauses run; default clause when none matched | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitSwitchStatement@L4410-L4542` | +| `DataStatementAst` (`VisitDataStatement`) | try/finally | sets context.LanguageMode to RestrictedLanguage around the body, runtime checks for allowed commands, assigns result to the data variable | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitDataStatement@L4649-L4729` | +| `ForEachStatementAst` (`VisitForEachStatement`) | loop | GenerateIteratorStatement with `$foreach`; enumerator via PSEnumerableBinder; body assigns the loop variable then runs the body | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitForEachStatement@L5058-L5088` | +| `DoWhileStatementAst` (`VisitDoWhileStatement`) | loop | GenerateDoLoop: body, then test; jump back while true | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitDoWhileStatement@L5123-L5126` | +| `DoUntilStatementAst` (`VisitDoUntilStatement`) | loop | GenerateDoLoop with the test negated | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitDoUntilStatement@L5128-L5131` | +| `ForStatementAst` (`VisitForStatement`) | loop | initializer; GenerateWhileLoop with condition and iterator as the `continue` action | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitForStatement@L5133-L5161` | +| `WhileStatementAst` (`VisitWhileStatement`) | loop | GenerateWhileLoop | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitWhileStatement@L5163-L5170` | +| `CatchClauseAst` (`VisitCatchClause`) | not visited | assert false; catch bodies are compiled by VisitTryStatement | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitCatchClause@L5172-L5176` | +| `TryStatementAst` (`VisitTryStatement`) | try/catch/finally | ExpressionType.Try; catch-all-only form catches RuntimeException (PipelineStoppedException rethrown), typed catches use ExceptionHandlingOps.FindMatchingHandler + switch; `$_` set to the ErrorRecord; finally wraps SuspendStoppingPipeline/RestoreStoppingPipeline | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTryStatement@L5247-L5509` | +| `BreakStatementAst` (`VisitBreakStatement`) | goto or throw | GenerateBreakOrContinue: Expression.Break to the matching compiled loop label, else throw BreakException(label) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitBreakStatement@L5553-L5561` | +| `ContinueStatementAst` (`VisitContinueStatement`) | goto or throw | same, with ContinueException | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitContinueStatement@L5563-L5571` | +| `ReturnStatementAst` (`VisitReturnStatement`) | return label | writes the value to the output pipe (or return pipe in class methods) then Expression.Return to the block's return label; inside a trap lambda throws ReturnException | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitReturnStatement@L5573-L5627` | +| `ExitStatementAst` (`VisitExitStatement`) | throw | throws the exception from PipelineOps.GetExitException(code) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitExitStatement@L5629-L5641` | +| `ThrowStatementAst` (`VisitThrowStatement`) | throw | ExceptionHandlingOps.ConvertToException(value, extent, isRethrow); `throw` with no value rethrows the exception being handled | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitThrowStatement@L5643-L5659` | +| `TernaryExpressionAst` (`VisitTernaryExpression`) | conditional | Expression.Condition(cond as bool, ifTrue as object, ifFalse as object) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTernaryExpression@L5677-L5685` | +| `BinaryExpressionAst` (`VisitBinaryExpression`) | operator call or dynamic site | constant-folded when constant; -and/-or -> AndAlso/OrElse over bool; double,double + - * / native; other arithmetic and comparisons via DynamicExpression with PSBinaryOperationBinder; -like/-match/-replace/-split/-join/-contains/-in/-is/-as/-f/.. -> ParserOps/StringOps/TypeOps helper calls; ?? -> Coalesce | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitBinaryExpression@L5687-L5977` | +| `UnaryExpressionAst` (`VisitUnaryExpression`) | operator call or dynamic site | constant-folded when constant; -not/! PSUnaryOperationBinder; unary +/- as 0+x / 0-x; -bnot OnesComplement; ++/-- CompileIncrementOrDecrement; unary -join/-split helper calls | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitUnaryExpression@L5994-L6051` | +| `ConvertExpressionAst` (`VisitConvertExpression`) | conversion | constant-folded; [ordered]@{} -> OrderedDictionary; [pscustomobject]@{} -> converter; [ref]$x -> VariableOps.GetVariableAsRef; otherwise ConvertValue: Expression.Convert or PSConvertBinder, PSDynamicConvertBinder when the type is unresolved at compile time | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitConvertExpression@L6102-L6164` | +| `ConstantExpressionAst` (`VisitConstantExpression`) | constant | Expression.Constant(value) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitConstantExpression@L6166-L6169` | +| `StringConstantExpressionAst` (`VisitStringConstantExpression`) | constant | Expression.Constant(value) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitStringConstantExpression@L6171-L6174` | +| `SubExpressionAst` (`VisitSubExpression`) | captured block | no statements -> $null; else CaptureAstResults (temporary output pipe collecting results) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitSubExpression@L6176-L6193` | +| `UsingExpressionAst` (`VisitUsingExpression`) | runtime call | VariableOps.GetUsingValue(locals, key, RuntimeUsingIndex, context) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitUsingExpression@L6195-L6203` | +| `VariableExpressionAst` (`VisitVariableExpression`) | slot or runtime lookup | $null/$true/$false constants; automatic variables via tuple slot or VariableOps.GetAutomaticVariableValue; TupleIndex < 0 -> VariableOps.GetVariableValue (by name); else tuple field access | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitVariableExpression@L6205-L6250` | +| `TypeExpressionAst` (`VisitTypeExpression`) | constant or runtime call | reflection Type constant if resolvable at compile time, else TypeOps.ResolveTypeName at run time | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTypeExpression@L6276-L6279` | +| `MemberExpressionAst` (`VisitMemberExpression`) | field/property or dynamic site | static [T]::Member with a unique public static field/property -> direct access; else PSGetMemberBinder / PSGetDynamicMemberBinder; `?.` wrapped in a null check | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitMemberExpression@L6281-L6346` | +| `InvokeMemberExpressionAst` (`VisitInvokeMemberExpression`) | dynamic site | InvokeMember: PSInvokeMemberBinder, PSCreateInstanceBinder for `[T]::new`, PSInvokeDynamicMemberBinder for non-constant names; argument/target type constraints passed to the binder | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitInvokeMemberExpression@L6447-L6469` | +| `ArrayExpressionAst` (`VisitArrayExpression`) | captured block | @( ) : empty -> object[0]; single pure expression -> that expression wrapped to array; else CaptureAstResults(Enumerable) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitArrayExpression@L6471-L6520` | +| `ArrayLiteralAst` (`VisitArrayLiteral`) | array init | Expression.NewArrayInit(object, elements cast to object) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitArrayLiteral@L6522-L6532` | +| `HashtableAst` (`VisitHashtable`) | block | new Hashtable(count, OrdinalIgnoreCase) then HashtableOps.AddKeyValuePair per pair; each value statement captured with CaptureStatementResults | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitHashtable@L6555-L6562` | +| `ScriptBlockExpressionAst` (`VisitScriptBlockExpression`) | runtime call | ScriptBlockExpressionWrapper.GetScriptBlock(context, false); the block body is compiled lazily on first run | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitScriptBlockExpression@L6564-L6577` | +| `ParenExpressionAst` (`VisitParenExpression`) | captured pipeline | assignment -> CompileAssignment; otherwise CaptureStatementResults of the pipeline | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitParenExpression@L6579-L6598` | +| `ExpandableStringExpressionAst` (`VisitExpandableStringExpression`) | format call | StringOps.FormatOperator(format string, string[] of nested expressions converted with PSToStringBinder) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitExpandableStringExpression@L6600-L6609` | +| `IndexExpressionAst` (`VisitIndexExpression`) | dynamic site | PSGetIndexBinder with 1 or n index arguments; `?[]` wrapped in a null check | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitIndexExpression@L6611-L6640` | +| `AttributedExpressionAst` (`VisitAttributedExpression`) | delegate | child.Accept(this); attributes only matter on assignment targets | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitAttributedExpression@L6650-L6653` | +| `BlockStatementAst` (`VisitBlockStatement`) | not compiled | returns null (workflow remnants) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitBlockStatement@L6655-L6658` | + +Rows: **63** (`GENERATED`). "not visited" rows are asserted unreachable in the compiler (their parents handle them). + +## 4. Helper methods of Compiler + +| Method | Source | +|---|---| +| `Compile` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964` | +| `CompileExpressionOperand` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileExpressionOperand@L966-L975` | +| `CompileInvocationArguments` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileInvocationArguments@L977-L991` | +| `ReduceAssignment` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ReduceAssignment@L993-L1023` | +| `Coalesce` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Coalesce@L1025-L1051` | +| `GetLocal` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetLocal@L1053-L1062` | +| `CallGetVariable` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CallGetVariable@L1064-L1071` | +| `CallSetVariable` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CallSetVariable@L1073-L1080` | +| `GetAutomaticVariable` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetAutomaticVariable@L1082-L1116` | +| `CallStringEquals` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CallStringEquals@L1118-L1126` | +| `IsStrictMode` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.IsStrictMode@L1128-L1136` | +| `AddSequencePoint` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.AddSequencePoint@L1138-L1149` | +| `UpdatePosition` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.UpdatePosition@L1151-L1163` | +| `NewTemp` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewTemp@L1167-L1170` | +| `GetTypeConstraintForMethodResolution` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetTypeConstraintForMethodResolution@L1172-L1192` | +| `CombineTypeConstraintForMethodResolution` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CombineTypeConstraintForMethodResolution@L1194-L1202` | +| `CombineTypeConstraintForMethodResolution` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CombineTypeConstraintForMethodResolution@L1204-L1217` | +| `ConvertValue` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ConvertValue@L1219-L1242` | +| `ConvertValue` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ConvertValue@L1244-L1256` | +| `GetParameterMetaData` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetParameterMetaData@L1262-L1294` | +| `GetAttributeGenerator` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetAttributeGenerator@L1299-L1320` | +| `CheckNoPositionalArgs` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CheckNoPositionalArgs@L1345-L1359` | +| `CheckNoNamedArgs` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CheckNoNamedArgs@L1361-L1376` | +| `static` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.static@L1378-L1386` | +| `NewCmdletBindingAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewCmdletBindingAttribute@L1388-L1444` | +| `NewExperimentalAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewExperimentalAttribute@L1446-L1463` | +| `NewParameterAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewParameterAttribute@L1465-L1548` | +| `NewOutputTypeAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewOutputTypeAttribute@L1550-L1628` | +| `NewAliasAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewAliasAttribute@L1630-L1643` | +| `NewValidateSetAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewValidateSetAttribute@L1645-L1710` | +| `NewDebuggerHiddenAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewDebuggerHiddenAttribute@L1712-L1717` | +| `NewValidateNotNullOrEmptyAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewValidateNotNullOrEmptyAttribute@L1719-L1724` | +| `NewValidateNotNullAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.NewValidateNotNullAttribute@L1726-L1731` | +| `GetAttribute` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetAttribute@L1733-L1796` | +| `GetAttribute` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetAttribute@L1798-L1817` | +| `GetRuntimeDefinedParameter` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetRuntimeDefinedParameter@L1819-L1907` | +| `TryGetDefaultParameterValue` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.TryGetDefaultParameterValue@L1909-L1943` | +| `Compile` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L1974-L2077` | +| `CompileTree` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2097` | +| `GetExpressionValue` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetExpressionValue@L2099-L2102` | +| `GetExpressionValue` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetExpressionValue@L2104-L2110` | +| `GetExpressionValue` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetExpressionValue@L2112-L2202` | +| `CompileSingleExpression` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleExpression@L2204-L2226` | +| `CaptureAstResults` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureAstResults@L2264-L2355` | +| `CaptureStatementResultsHelper` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureStatementResultsHelper@L2357-L2397` | +| `CaptureStatementResults` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureStatementResults@L2399-L2426` | +| `CallAddPipe` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CallAddPipe@L2428-L2436` | +| `CompileNamedBlock` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileNamedBlock@L2504-L2527` | +| `CompileTrap` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTrap@L2529-L2550` | +| `CompileSingleLambda` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleLambda@L2552-L2626` | +| `GenerateTypesAndUsings` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateTypesAndUsings@L2628-L2675` | +| `GenerateLoadUsings` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateLoadUsings@L2682-L2708` | +| `DefinePowerShellTypes` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.DefinePowerShellTypes@L2717-L2750` | +| `LoadUsingsImpl` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.LoadUsingsImpl@L2752-L2796` | +| `PopulateRuntimeTypes` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.PopulateRuntimeTypes@L2798-L2820` | +| `LoadAssembly` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.LoadAssembly@L2822-L2854` | +| `LoadModule` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.LoadModule@L2861-L2901` | +| `GenerateFunctionProlog` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateFunctionProlog@L2903-L2959` | +| `InitializeMemberProperties` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.InitializeMemberProperties@L2961-L2985` | +| `GenerateFunctionEpilog` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateFunctionEpilog@L2987-L2993` | +| `CompileStatementListWithTraps` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268` | +| `CompileTrappableExpression` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTrappableExpression@L3270-L3279` | +| `ShouldSetExecutionStatusToSuccess` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ShouldSetExecutionStatusToSuccess@L3287-L3299` | +| `ShouldSetExecutionStatusToSuccess` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ShouldSetExecutionStatusToSuccess@L3307-L3319` | +| `GetSingleExpressionFromPipeline` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetSingleExpressionFromPipeline@L3326-L3335` | +| `ShouldSetExecutionStatusToSuccess` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ShouldSetExecutionStatusToSuccess@L3343-L3364` | +| `ShouldSetExecutionStatusToSuccess` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ShouldSetExecutionStatusToSuccess@L3372-L3419` | +| `CompileAssignment` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileAssignment@L3527-L3563` | +| `CompilePipelineChainElement` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompilePipelineChainElement@L3726-L3734` | +| `GetCommandRedirections` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetCommandRedirections@L3868-L3894` | +| `GetRedirectedExpression` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetRedirectedExpression@L3899-L4110` | +| `AddMergeRedirectionExpressions` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.AddMergeRedirectionExpressions@L4112-L4138` | +| `GetCommandArgumentExpression` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetCommandArgumentExpression@L4239-L4266` | +| `ThrowRuntimeError` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ThrowRuntimeError@L4307-L4310` | +| `ThrowRuntimeError` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ThrowRuntimeError@L4312-L4315` | +| `ThrowRuntimeError` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ThrowRuntimeError@L4317-L4335` | +| `ThrowRuntimeErrorWithInnerException` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ThrowRuntimeErrorWithInnerException@L4337-L4344` | +| `ThrowRuntimeErrorWithInnerException` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.ThrowRuntimeErrorWithInnerException@L4346-L4370` | +| `CreateThrow` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CreateThrow@L4372-L4392` | +| `CreateThrow` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CreateThrow@L4394-L4408` | +| `GetSwitchBodyGenerator` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetSwitchBodyGenerator@L4544-L4647` | +| `GenerateLoopBreakContinueCatchBlocks` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateLoopBreakContinueCatchBlocks@L4731-L4758` | +| `GenerateWhileLoop` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateWhileLoop@L4760-L4864` | +| `GenerateDoLoop` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateDoLoop@L4866-L4929` | +| `GenerateIteratorStatement` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateIteratorStatement@L4931-L5056` | +| `GetRangeEnumerator` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetRangeEnumerator@L5090-L5108` | +| `UpdatePositionForInitializerOrCondition` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.UpdatePositionForInitializerOrCondition@L5110-L5121` | +| `GenerateBreakOrContinue` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateBreakOrContinue@L5511-L5551` | +| `GenerateCallContains` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateCallContains@L5665-L5675` | +| `GetLikeRHSOperand` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetLikeRHSOperand@L5979-L5992` | +| `CompileIncrementOrDecrement` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileIncrementOrDecrement@L6053-L6100` | +| `CompileTypeName` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTypeName@L6252-L6274` | +| `GetInvokeMemberConstraints` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetInvokeMemberConstraints@L6348-L6376` | +| `GetInvokeMemberConstraints` (overload) | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetInvokeMemberConstraints@L6378-L6403` | +| `InvokeMember` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.InvokeMember@L6405-L6423` | +| `InvokeBaseCtorMethod` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.InvokeBaseCtorMethod@L6425-L6430` | +| `InvokeDynamicMember` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.InvokeDynamicMember@L6432-L6445` | +| `BuildHashtable` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.BuildHashtable@L6534-L6553` | +| `GetNullConditionalWrappedExpression` | `src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GetNullConditionalWrappedExpression@L6642-L6648` | + +Verification: all 63 `Visit*` methods were read (notes W3-W21); the table is joined by name and the generator asserts that every `Visit*` method has an annotation and vice versa; `VisitPipeline`, `VisitTrap`, `VisitHashtable`, `VisitCatchClause`, `VisitDoWhileStatement` were re-read for this catalog. + diff --git a/docs-out/02-language/EQUIVALENCE-SPEC.md b/docs-out/02-language/EQUIVALENCE-SPEC.md new file mode 100644 index 0000000..1716f83 --- /dev/null +++ b/docs-out/02-language/EQUIVALENCE-SPEC.md @@ -0,0 +1,134 @@ +--- +id: LANG-EQUIV-001 +title: Equivalence specification +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/interpreter/Interpreter.cs, src/System.Management.Automation/engine/interpreter/LightCompiler.cs, src/System.Management.Automation/engine/interpreter/LoopCompiler.cs, src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs, src/System.Management.Automation/engine/parser/Compiler.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/interpreter/_folder.md] +generated_from: [] +--- + +# Equivalence specification + +> STATIC-ONLY — behavior not run-verified. No oracle (`pwsh`) ran, so every statement is `FACT-SRC` about *how the code is built to be equivalent* or `INFERENCE`/`UNVERIFIED` about whether it is. Related: [TIERING-POLICY.md](TIERING-POLICY.md), [IR-SPEC.md](IR-SPEC.md), [INSTRUCTION-SET.md](INSTRUCTION-SET.md), [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md). + +## 1. Purpose and scope [SPEC] + +The same IR runs in two ways: through the instruction interpreter and as JIT-compiled code, and a call can switch from one to the other at any call boundary (and a running loop can switch mid-execution). Scripts must not be able to tell. This document lists the **equivalence obligations** between the two tiers and the optimized/unoptimized variants, how the code meets each (or fails to), and what a new implementation with a single tree-walking tier must preserve. + +## 2. Source map + +| path | role | +|---|---| +| src/System.Management.Automation/engine/interpreter/Interpreter.cs::Interpreter@L31-L109 | instruction loop | +| src/System.Management.Automation/engine/interpreter/InterpretedFrame.cs::InterpretedFrame@L28-L332 | frame: locals, stack, continuations | +| src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterTryCatchFinallyInstruction@L305-L447 | interpreted try/catch/finally | +| src/System.Management.Automation/engine/interpreter/LoopCompiler.cs::LoopCompiler@L29-L384 | hand-over of a running loop to JIT code | +| src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda@L43-L302 | tier switch at call boundaries | + +## 3. Responsibilities and non-responsibilities + +Specifies what must be equal; it does not choose the tier ([TIERING-POLICY.md](TIERING-POLICY.md)). + +## 4. Key types and contracts [IMPL] + +Both tiers consume the same `Expression>`. The interpreter compiles it to `InstructionArray` once and runs it per call with an `InterpretedFrame`; the JIT form is `LambdaExpression.Compile()` of the same tree (for lambdas with closure variables the `LightLambdaClosureVisitor` rewrites captured variables; `Compiler` produces none, `INFERENCE`). + +## 5. Data model — equivalence obligations + +| ID | Obligation | How the code meets it | Status | +|---|---|---|---| +| EQ-01 | Same result and output order for every statement | one IR; instructions and LINQ nodes implement the same operations; dynamic sites use the same binders in both tiers | `FACT-SRC` (design) | +| EQ-02 | Same binding rules for operators, members, conversions | `Expression.Dynamic` becomes a `DynamicInstruction<…>` holding its own `CallSite`; binding rules come from the PowerShell binders, not from either tier | `FACT-SRC`; per-site caches are separate per tier (performance only) | +| EQ-03 | Same exception behavior: type, message, catch order, rethrow identity | interpreter `try/catch/finally` instruction uses CLR `try/catch/finally` inside the instruction and picks the first handler assignable from the exception type, as the CLR does; `RethrowException` marks a rethrow; stack trace is saved into `Exception.Data` | `FACT-SRC` code shape; stack-trace text differs (interpreted frames are reported through `InterpretedFrameInfo`) | +| EQ-04 | `finally` runs on `return`/`break`/`continue`/exception | interpreter uses continuation stacks (`EnterFinally`/`LeaveFinally`, pending continuation) for gotos leaving a `try`; JIT uses the CLR | `FACT-SRC`; equivalence untested here | +| EQ-05 | Flow-control exceptions pass through catch-alls | the IR catches `FlowControlException` first and rethrows; the interpreter lets `RethrowException` pass | `FACT-SRC` | +| EQ-06 | Same `$?`, same output pipe contents | `$?` and pipe writes are explicit IR operations (calls/field stores), not tier features | `FACT-SRC` | +| EQ-07 | Same error positions and breakpoint behavior | `UpdatePositionExpr` reduces to `DebugInfo` + store (JIT) or `UpdatePositionInstruction` (interpreter); both store the sequence-point index and call `OnSequencePointHit` if debugging | `FACT-SRC` | +| EQ-08 | Same variable visibility | locals tuple is a heap object in both tiers; the tier does not change slots | `FACT-SRC` | +| EQ-09 | Value-type receivers are mutated in place | interpreter's reflection field store works on a boxed copy, so the light compiler forces JIT for instance calls on non-primitive value types and by-ref calls | `FACT-SRC` (workaround for a known interpreter limit) | +| EQ-10 | Mid-loop promotion keeps state | `LoopCompiler` copies variables in from the frame (`Data`, closure boxes), runs the loop as JIT code, writes modified variables back in a `finally`, and exits via `frame.Goto` for jumps leaving the loop; the evaluation stack is not transferred | `FACT-SRC`; correctness of write-back for all shapes `UNVERIFIED` | +| EQ-11 | Arithmetic and comparison on primitive types | the IR uses native typed operators only for `double`; everything else is a dynamic site, so the typed interpreter instructions are not reached from PowerShell IR | `INFERENCE` (see defects below) | +| EQ-12 | Thread safety of promotion | counters race benignly; compile takes a lock and runs once; published delegate used by later calls | `FACT-SRC` | + +## 6. Control flow + +Not applicable: this document specifies properties, not a flow. The tier-switch points are: (1) call boundary (`LightLambda.TryGetCompiled`), (2) loop-top arrival (`EnterLoopInstruction`). + +## 7. Algorithms + +### 7.1 Known non-equivalences and defects (reading only, not run) + +- **Interpreter instruction defects.** `AddInstruction`, `SubInstruction`, `MulInstruction`, `DivInstruction` (and their overflow variants) cast `UInt64` operands to `Int16` in the `UInt64` nested class (sites: `AddInstruction.cs` L98 and L231, same lines in `Sub`/`Mul`, `DivInstruction.cs` L98); `EqualInstruction.Create`/`NotEqualInstruction.Create` map the unsigned integer types to the signed classes; `EqualInstruction.Create(typeof(string))` falls through to `NotImplementedException`. These classes are inherited DLR code; the compiler emits typed arithmetic only for `double` (`Compiler.cs` L5742-L5771), so they should be unreachable (`INFERENCE`, VB-016). +- **Unsupported shapes** throw `NotImplementedException` in the light compiler ([IR-SPEC.md](IR-SPEC.md) §7.2). +- **`LessThanOrEqual`/`GreaterThanOrEqual`** have no instruction (`InstructionList` throws `NotSupportedException`). +- **Nondeterministic tier** (async promotion): only performance and debugger-visible differences are allowed. +- **Exception stack traces** differ in shape between tiers. + +### 7.2 Obligations for a single-tier tree-walker (the MicroteamShell case) + +A tree-walking evaluator inherits EQ-01..EQ-08 automatically if it implements the protocols of [IR-SPEC.md](IR-SPEC.md) §5 once. It needs no EQ-09..EQ-12. It must still decide: evaluation order (arguments left to right; for assignments the right side before the left target's member/index operands); that `finally` runs on every exit; that flow-control signals are not swallowed by catch-all handling; that each statement updates the current sequence point before evaluation. + +## 8. Observable behavior [SPEC] + +None directly. Verification plan: run each corpus snippet with the call counter forced low and forced high (`UNVERIFIED`, needs a hook not present in release builds), compare output, error records and `$?`. + +## 9. Error handling and edge cases + +A compile failure of the JIT form on the pool thread leaves the interpreted form running; whether the failure is logged was not found (`UNVERIFIED`). + +## 10. Concurrency and threading + +EQ-12. A call may run interpreted while the same lambda is being compiled on another thread; both forms read the same `FunctionContext` types and heap locals. + +## 11. Extension points and customization hooks + +`IInstructionProvider` lets an extension node supply interpreter instructions directly; every new extension node must also have a correct `Reduce()` for the JIT (the two must stay equivalent). + +## 12. Dependencies + +`System.Linq.Expressions` (JIT), the interpreter folder, the PowerShell binders. + +## 13. Performance notes + +Equivalence is the reason for forced JIT on by-ref/value-type calls: correctness over start-up speed. + +## 14. Security considerations (mandatory) + +Security checks (language mode, AMSI, logging) sit outside both tiers (in `InvokeWithPipeImpl` and `PerformSecurityChecks`), so tier choice cannot bypass them; the debugger-induced switch to unoptimized code does not change enforcement (`FACT-SRC`, [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §14). + +## 15. Enterprise considerations + +No differential-test harness for tier equivalence was found in the source tree (`UNVERIFIED`); this is the main testability gap. + +## 16. Tests + +None located for tier parity (VB-014). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| keep one IR for both tiers | `FACT-SRC` | equivalence by construction for ordinary nodes | +| work around interpreter limits by forcing JIT | `FACT-SRC` | unsupported cases cost start-up time instead of correctness | +| inherit DLR interpreter wholesale | `INFERENCE` | dead and defective instruction classes remain | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-EQV-01** [REPLICATE] (area: NLD-RUN) Define behavior once, in the evaluator and the protocols, never per execution strategy; if two strategies are ever added, test them differentially. Evidence: src/System.Management.Automation/engine/interpreter/Interpreter.cs::Interpreter@L31-L109. +- **LSN-EQV-02** [REPLICATE] (area: NLD-ERR) `finally` must run for every exit (return, break, continue, throw) and flow-control signals must never be swallowed by catch-all handlers. Evidence: src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterTryCatchFinallyInstruction@L305-L447. +- **LSN-EQV-03** [AVOID] (area: NLD-RUN) Do not port the typed arithmetic/comparison instruction families or the DLR-derived interpreter; they are unreachable or defective for PowerShell. Replace with the evaluator's own numeric semantics from CONVERSION-RULES (planned T28). Evidence: src/System.Management.Automation/engine/interpreter/Instruction.cs::Instruction@L27-L70. DEV: DEV-EQV-061 + +## 19. Open questions / UNVERIFIED + +- VB-016: confirm by a search of `Compiler.cs` and binder code that no path creates a typed instruction for `UInt64`/`string`. +- Whether mid-loop promotion preserves all `try/finally` states entered before the loop (the stack is not transferred). +- No oracle: all of EQ-01..EQ-12 remain design claims. + +Verification: `ControlFlowInstructions.cs`, `LoopCompiler.cs`, `LightLambda.cs`, `LightCompiler.cs`, `InstructionList.cs`, the arithmetic instruction files and `Utilities.cs` were read in full (interpreter notes); defect sites re-checked in the notes with line numbers; `Compiler.cs` arithmetic emission L5742-L5771 cross-read. diff --git a/docs-out/02-language/ERROR-CATALOG.md b/docs-out/02-language/ERROR-CATALOG.md new file mode 100644 index 0000000..312dd2f --- /dev/null +++ b/docs-out/02-language/ERROR-CATALOG.md @@ -0,0 +1,471 @@ +--- +id: LANG-ERRCAT-001 +title: Parser and tokenizer error catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-ERR-002] +sources: [src/System.Management.Automation/resources/ParserStrings.resx, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/tokenizer.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/resources/_folder.md] +generated_from: [data/resources.json, data/parser-methods.json] +--- + +# Parser and tokenizer error catalog + +> STATIC-ONLY — behavior not run-verified. Every row is `GENERATED` by `docs-out/tools/gen_T22.py` (`APPROX`: use sites come from `data/resources.json`, the (E)/(I) classification looks at the nearest preceding `ReportError(` / `ReportIncompleteInput(` call in the source). Narrative: [02.2-parser.md](02.2-parser.md) §9, [02.1-tokenizer.md](02.1-tokenizer.md) §9. + +## 1. Purpose and scope [SPEC] + +Catalog of the parse error IDs. An error ID is the resource key of `ParserStrings.resx` (the code passes `nameof(ParserStrings.Key)` as ID and `ParserStrings.Key` as message template; `FACT-SRC`, `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8143-L8166`). `ParserStrings.resx` holds **412** keys; 157 are reported from `Parser.cs` or `tokenizer.cs` (section 3); the others are reported by semantic checks, type/class building or other code (section 4, owned by [02.4-semantic-analysis.md](02.4-semantic-analysis.md), planned T24). + +Report kinds: **(E)** `ReportError` — a plain error; **(I)** `ReportIncompleteInput` — the error sets `ParseError.IncompleteInput` when the error position is at the end of the script (so a REPL can ask for another line) (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8111-L8141`); **(?)** the classifier found neither call nearby (error built through a variable, e.g. `errorId`/`errorMsg`). + +Dedup: `SaveError` drops an error when an earlier error has the same ID and the same start/end line and column; there is no cap on the number of errors (`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057`). + +**Parse-error limits and recovery rule.** The parser never throws for a syntax error: each error is appended to `Parser.ErrorList` after de-duplication (an error is dropped only if an earlier error has the same ID and the same start and end line and column), there is no limit on the number of errors, and parsing continues by returning partial or error nodes (`ErrorStatementAst`, `ErrorExpressionAst`), by pretending a missing closing token was present, or by skipping tokens to a synchronization token (`SyncOnError`, which tracks nesting of `()`, `{}` and `[]` and never consumes end of input); `ParseError.IncompleteInput` is true only when the error was reported with `ReportIncompleteInput` and the error position is at the end of the script apart from whitespace and comments. The only exceptions are an exhausted execution stack (the whole parse is retried once on a new thread, then `ScriptTooComplicated` is reported) and any other exception during `ParseFile`/`ParseInput`, which is wrapped as a `ParseException` with `UnrecoverableParserError`. + +## 2. Source map + +| path | type | role | +|---|---|---| +| `src/System.Management.Automation/resources/ParserStrings.resx` | resx | error message templates | +| `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportError@L8143-L8166` | method | report a plain error | +| `src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8111-L8141` | method | report an error that may mean 'more input expected' | +| `src/System.Management.Automation/engine/parser/Parser.cs::ParseError@L8193-L8241` | class | public error object: `Extent`, `ErrorId`, `Message`, `IncompleteInput` | + +## 3. Keys reported by Parser.cs or tokenizer.cs + +| Error ID (resource key) | Message template | Parser.cs lines | tokenizer.cs lines | Also used in | +|---|---|---|---|---| +| `AllStream` | * stream | L6245(?) | | | +| `ArrayHasTooManyDimensions` | The array has too many dimensions: {0}. The number of dimensions for an array must be less than or equal to 32. | L1554(E) | | | +| `BackgroundOperatorInPipelineChain` | Background operators can only be used at the end of a pipeline chain. | L5934(E) | | | +| `BadNumericConstant` | The numeric constant {0} is not valid. | | L3912(E) | | +| `CannotLoadAssemblyFromUncPath` | Cannot use assembly with an UNC path: '{0}'. | L5099(E) | | | +| `CannotLoadAssemblyWithUriSchema` | Cannot use assembly with uri schema '{0}'. | L5108(E) | | | +| `ClassesNotAllowedInConstrainedLanguage` | Class keyword is not allowed in ConstrainedLanguage mode. | L4231(E) | | | +| `ConfigurationBodyEmpty` | The configuration block results were null or empty. Verify that configurations were defined in the block. | L3137(E) | | | +| `ConfigurationNotAllowedInConstrainedLanguage` | Configuration keyword is not allowed in constrainedLanguage mode. | L2993(E) | | | +| `ConfigurationNotAllowedOnArm64` | Configuration keyword is not supported on ARM64 processors. | L3012(E) | | | +| `ConfigurationNotAllowedOnWinPE` | Configuration is not supported on WinPE. | L3022(E) | | | +| `DebugStream` | debug stream | L6250(?) | | | +| `DuplicateNamedArgument` | Duplicate named arguments '{0}' are not allowed. | L1225(E) | | `engine/parser/SemanticChecks.cs` | +| `DuplicateQualifier` | Duplicate '{0}' qualifier | L4455(E) L4477(E) L4500(E) | | | +| `DuplicateScriptCommandClause` | Script command clause '{0}' has already been defined. | L1836(E) | | | +| `DynamicKeywordPostParseException` | Exception occurred when post-parsing dynamic keyword '{0}' with details '{1}'. | L4081(E) | | | +| `DynamicKeywordPreParseException` | Exception occurred when pre-parsing dynamic keyword '{0}' with details '{1}'. | L3745(E) | | | +| `EmptyPipeElement` | An empty pipe element is not allowed. | L6083(I) L6136(I) | | | +| `EmptyPipelineChainElement` | A pipeline chain operator must be followed by a pipeline. | L5888(I) L5902(E) L5982(I) | | | +| `EmptyVariableReference` | An empty ${} variable reference was found. A name is required inside the braces. | | L2968(E) | | +| `EndSquareBracketExpectedAtEndOfAttribute` | Missing ] at end of attribute or type literal. | L1127(I) L1147(E) L1485(I) | | | +| `EndSquareBracketExpectedAtEndOfType` | Missing ] at end of type token. | L1426(I) L7830(I) | | | +| `ErrorLoadingAssembly` | Cannot load assembly '{0}'. | L5157(E) | | `engine/parser/Compiler.cs`, `src/Microsoft.PowerShell.Commands.Utility/commands/utility/AddType.cs` | +| `ErrorStream` | error stream | L6247(?) | | | +| `ExpectedExpression` | An expression was expected after '('. | L7594(I) | | | +| `ExpectedValueExpression` | You must provide a value expression following the '{0}' operator. | L4857(E) L5831(I) L6739(I) | | | +| `ExpressionsMustBeFirstInPipeline` | Expressions are only allowed as the first element of a pipeline. | L6038(E) | | | +| `FileReadError` | The file could not be read: {0}. | L89(?) | | `engine/parser/Compiler.cs` | +| `IfStatementMissingCondition` | Missing condition in if statement after '{0} ('. | L2444(I) | | | +| `IncompleteDollarSubexpressionReference` | $(subexpression) is missing the closing ')'. | | L2416(I) | | +| `IncompleteDollarVariableReference` | ${{variable}} reference starting is missing the closing '}}'. | | L2959(I) | | +| `IncompleteMemberDefinition` | Missing a property name or method definition. | L4632(I) | | | +| `IncompletePropertyAssignmentBlock` | There is an incomplete property assignment block in the instance definition. | L7383(?) | | | +| `IncompleteString` | Incomplete string token. | | L4710(I) | | +| `InformationStream` | information stream | L6251(?) | | | +| `InvalidBracedVariableReference` | Variable reference is not valid. The variable name is missing. | | L3000(E) | | +| `InvalidConfigurationName` | The configuration name '{0}' is not valid. Standard names may only contain letters (a-z, A-Z), numbers (0-9), period (.), hyphen (-) and underscore (_). The name may not be null or empty, and should start with a letter. | L2960(E) | | `DscSupport/CimDSCParser.cs` | +| `InvalidForeachFlag` | The parameter {0} is not valid for the foreach statement. | L3411(E) | | | +| `InvalidFunctionParameter` | Parameter declarations are a comma-separated list of variable names with optional initializer expressions. | L984(I) | | | +| `InvalidParameterForDataSectionStatement` | The "{0}" parameter of the Data section is not valid. The valid Data section parameter is SupportedCommand. | L5704(E) | | | +| `InvalidSwitchFlag` | The parameter {0} is not valid for the switch statement. | L2677(E) | | | +| `InvalidUnderlyingType` | '{0}' is not a valid underlying type for enums. Expected a builtin integral type (one of byte, sbyte, short, ushort, int, uint, long or ulong) | L4757(E) | | `engine/parser/PSType.cs` | +| `InvalidUnicodeEscapeSequence` | The Unicode escape sequence is not valid. A valid sequence is `u{ followed by one to six hex digits and a closing '}'. | | L1506(E) L1527(E) L1542(E) | | +| `InvalidUnicodeEscapeSequenceValue` | The Unicode escape sequence value is out of range. The maximum value is 0x10FFFF. | | L1584(E) | | +| `InvalidValueForUsingItemName` | '{0}' is not a valid value for using name. | L4973(E) L4985(E) L5044(E) | | | +| `InvalidVariableReference` | Variable reference is not valid. '$' was not followed by a valid variable name character. Consider using ${} to delimit the name. | | L3172(?) | | +| `InvalidVariableReferenceWithDrive` | Variable reference is not valid. ':' was not followed by a valid variable name character. Consider using ${} to delimit the name. | | L3167(?) | | +| `MissingArgument` | Missing argument in parameter list. | L6540(E) | | | +| `MissingArrayIndexExpression` | Array index expression is missing or not valid. | L7996(I) | | | +| `MissingAssemblyNameSpecification` | The type name is missing the assembly name specification. | L1386(E) L1511(E) L1605(E) | | | +| `MissingBaseCtorCall` | Missing 'base' after ':' in a base class constructor call. | L5245(I) | | | +| `MissingBraceInObjectDefinition` | Missing '{' in object definition. | L3804(I) L3921(I) | | | +| `MissingCatchHandlerBlock` | The Catch block is missing its statement block. | L5571(I) | | | +| `MissingCatchOrFinally` | The Try statement is missing its Catch or Finally block. | L5659(I) | | | +| `MissingColonInTernaryExpression` | Missing ':' in the ternary expression. | L6761(I) | | | +| `MissingConfigurationName` | The name for the configuration is missing. Provide the missing name as a simple name, string, or string-valued expression. | L2917(E) L2930(I) L2944(I) | | | +| `MissingCurlyBraceInSwitchStatement` | Missing '{' in switch statement. | L2775(I) | | | +| `MissingCurlyInConfigurationStatement` | Missing '{' in configuration statement. | L3116(I) | | | +| `MissingEndCurlyBrace` | Missing closing '}' in statement block or type definition. | L1645(I) L1886(I) L2874(I) | | | +| `MissingEndParenthesisAfterForeach` | Missing closing ')' after expression part of foreach loop. The correct form is: foreach ($a in $b) {...} | L3489(I) | | | +| `MissingEndParenthesisAfterStatement` | Missing closing ')' after expression in '{0}' statement. | L2466(I) L3600(I) L3685(I) | | | +| `MissingEndParenthesisInExpression` | Missing closing ')' in expression. | L1111(I) L7607(I) | | | +| `MissingEndParenthesisInFunctionParameterList` | Missing ')' in function parameter list. | L884(I) L5439(I) | | | +| `MissingEndParenthesisInMethodCall` | Missing ')' in method call. | L7954(I) | | | +| `MissingEndParenthesisInSubexpression` | Missing closing ')' in subexpression. | L7548(I) | | | +| `MissingEndParenthesisInSwitchStatement` | Missing ')' after expression in switch statement. | L2736(I) | | | +| `MissingEndSquareBracket` | Missing ']' after array index expression. | L8012(I) | | | +| `MissingEqualsInHashLiteral` | Missing '=' operator after key in hash literal. | L7446(?) | | | +| `MissingEqualsInPropertyAssignmentBlock` | Missing '=' operator after key in property assignment. | L7441(?) L7472(?) | | | +| `MissingEqualsInUsingAlias` | Missing '=' operator | L5070(I) | | | +| `MissingExpression` | Missing expression after '{0}' in pipeline element. | L6333(I) L6667(E) | | | +| `MissingExpressionAfterKeyword` | Missing expression after '{0}' in loop. | L3663(I) L4166(I) | | | +| `MissingExpressionAfterOperator` | Missing expression after unary operator '{0}'. | L7124(I) | | | +| `MissingExpressionAfterToken` | Missing expression after '{0}'. | L932(I) L1009(I) L1247(I) | | | +| `MissingExpressionInNamedArgument` | Missing statement after '=' in named argument. | L1197(I) | | | +| `MissingFileSpecification` | Missing file specification after redirection operator. | L6186(E) | | | +| `MissingFilenameOption` | Cannot process the 'switch' statement because of a missing file name argument to the -file option. | L2651(I) | | | +| `MissingFinallyStatementBlock` | The Finally block is missing its statement block. | L5647(I) | | | +| `MissingForeachExpression` | Missing expression in foreach loop. The correct form is: foreach ($a in $b) {...} | L3475(I) | | | +| `MissingForeachStatement` | Missing statement body in foreach loop. The correct form is: foreach ($a in $b) {...} | L3501(I) | | | +| `MissingFunctionBody` | Missing function body in function declaration. | L5270(I) L5382(I) | | | +| `MissingInInForeach` | Missing 'in' after variable in foreach loop. The correct form is: foreach ($a in $b) {...} | L3462(I) | | | +| `MissingLoopStatement` | Missing statement body in {0} loop. | L3612(I) L3700(I) L4123(I) | | | +| `MissingMethodParameterList` | Missing '(' in class method parameter list. | L5195(I) L5237(I) | | | +| `MissingNameAfterKeyword` | Missing name after {0} keyword. | L4252(I) L4725(I) L5359(I) | | | +| `MissingNamedBlocks` | unexpected token '{0}', expected 'begin', 'process', 'end', 'clean', or 'dynamicparam'. | L1769(E) | | | +| `MissingNamedStatementBlock` | Missing statement block after '{0}'. | L1800(I) | | | +| `MissingNamespaceAlias` | Missing namespace alias | L5015(I) | | | +| `MissingOpenParenthesisAfterKeyword` | Missing opening '(' after keyword '{0}'. | L3428(I) L3552(I) L3646(I) | | | +| `MissingOpenParenthesisInIfStatement` | Missing '(' after '{0}' in if statement. | L2428(I) | | | +| `MissingPropertyName` | Missing property name after reference operator. | L7734(I) | | | +| `MissingPropertyTerminator` | Missing ';' or end-of-line in property definition. | L4565(I) L4881(I) | | | +| `MissingStatementAfterKeyword` | Missing statement body after keyword '{0}'. | L2352(I) L2389(I) L4023(I) | | | +| `MissingStatementBlock` | Missing statement block after {0} ( condition ). | L2482(I) | | | +| `MissingStatementBlockAfterElse` | Missing statement block after 'else' keyword. | L2522(I) | | | +| `MissingStatementBlockForDataSection` | The Data section is missing its statement block. | L5756(I) | | | +| `MissingStatementInHashLiteral` | Missing statement after '=' in hash literal. | L7477(?) | | | +| `MissingStatementTerminator` | Missing a newline or semicolon. | L567(I) | | | +| `MissingSwitchConditionExpression` | Missing condition in switch statement clause. | L2806(I) | | | +| `MissingSwitchStatementClause` | Missing statement block in switch statement clause. | L2829(I) | | | +| `MissingTerminatorMultiLineComment` | The terminator '#>' is missing from the multiline comment. | | L1830(I) | | +| `MissingThrottleLimit` | The ThrottleLimit parameter of the foreach statement is missing a value. Supply a throttle limit to the parameter. | L3401(I) | | | +| `MissingTrapStatement` | The 'trap' statement was incomplete. A trap statement requires a body. | L5475(I) | | | +| `MissingTryStatementBlock` | The Try statement is missing its statement block. | L5618(I) | | | +| `MissingTypeBody` | Missing '{0}' body in '{0}' declaration. | L4311(I) L4774(I) | | | +| `MissingTypeLiteralToken` | Missing type literal. | L5526(I) | | | +| `MissingTypename` | Missing type name after '['. | L1077(I) L1371(I) L1411(I) | | | +| `MissingUnicodeEscapeSequenceTerminator` | The Unicode escape sequence is missing the closing '}'. | | L1548(E) | | +| `MissingUsingItemName` | Missing using name | L4963(I) | | | +| `MissingUsingStatementDirective` | Missing using directive | L4946(I) | | | +| `MissingValueForSupportedCommandInDataSectionStatement` | The SupportedCommand parameter of the Data section is missing a value. Supply a cmdlet or function name to the parameter. | L5723(I) | | | +| `MissingVariableNameAfterForeach` | Missing variable name after foreach. The correct form is: foreach ($a in $b) {...} | L3444(I) | | | +| `MissingWhileOrUntilInDoWhile` | Missing while or until keyword in do loop. | L4138(I) | | | +| `ModifiersCannotBeCombined` | Modifier '{0}' cannot be combined with '{1}' | L4463(E) L4485(E) | | | +| `MultipleSwitchDefaultClauses` | A switch statement can have only one default clause. | L2845(E) | | | +| `OpenBraceNeedsToBeBackTickedInVariableName` | Use `{ instead of { in variable names. | | L2937(E) | | +| `OutputStream` | output stream | L6246(?) | | | +| `ParameterCannotHaveArgument` | Parameter {0} cannot have an argument. | | L2018(E) | | +| `ParameterRequiresArgument` | Parameter {0} requires an argument. | L3350(E) L6567(E) | L2029(E) | | +| `ParserError` | {0} | L3096(E) L3283(E) | | | +| `PipelineValueRequired` | A switch statement must have one of the following: '-file file_name' or '( expression )'. | L2703(E) L2716(I) L2751(I) | | | +| `RedirectionNotSupported` | The '{0}' operator is reserved for future use. | L6195(E) L6217(E) L6228(E) | | | +| `RequiredNameOrExpressionMissing` | A required name or expression was missing. | L3340(I) L3797(I) L3819(E) | | | +| `RequiresArgumentMustBeConstant` | Argument must be constant. | | L2039(E) | `DscSupport/CimDSCParser.cs`, `engine/parser/SymbolResolver.cs` | +| `RequiresInvalidStringArgument` | The argument for the {0} parameter is not valid. Specify a valid string argument. | | L2059(E) L2072(E) L2221(E) | | +| `RequiresModuleInvalid` | The argument for the Module parameter is not valid. {0} | | L2190(E) L2198(E) | `engine/parser/SymbolResolver.cs` | +| `RequiresPSEditionInvalid` | The argument for the {0} parameter is not valid. Specify a valid PowerShell edition. | | L2255(E) | | +| `RequiresPSEditionValueIsAlreadySpecified` | The argument for the {0} parameter contains duplicate values. Do not specify duplicate PowerShell edition values. | | L2267(E) | | +| `RequiresVersionInvalid` | The argument for the Version parameter is not valid. Specify a valid PowerShell version, in the format major.minor version. | | L2129(E) | `DscSupport/CimDSCParser.cs` | +| `ReservedKeywordNotAllowed` | The '{0}' keyword is not supported in this version of the language. | L2090(E) | | | +| `ScriptTooComplicated` | Processing was stopped because the script is too complex. | L220(E) | | `engine/parser/SemanticChecks.cs` | +| `StreamAlreadyRedirected` | The {0} for this command is already redirected. | L6257(E) | | | +| `TerminatorExpectedAtEndOfString` | The string is missing the terminator: {0}. | | L2346(I) L2508(I) L2599(I) | | +| `TokenAfterEndOfValidScriptText` | Executable script code found in signature block. | | L1141(E) | | +| `TooManyDigitsInUnicodeEscapeSequence` | The Unicode escape sequence contains more than the maximum of six hex digits between braces. | | L1560(E) | | +| `TooManyTypes` | Only one type may be specified on class members. | L4439(E) | | | +| `TypeNameExpected` | Type name expected. | L4283(I) L4747(I) | | | +| `TypeNotAllowedBeforeParam` | Unexpected type [{0}] was found in the parameter statement. | L904(E) | | | +| `TypeNotAllowedBeforeStatement` | Unexpected type [{0}] was found before statement. | L2016(E) | | | +| `UnexpectedAttribute` | Unexpected attribute '{0}'. | L2006(E) L7153(I) | | `engine/parser/SemanticChecks.cs` | +| `UnexpectedCharactersAfterHereStringHeader` | No characters are allowed after a here-string header but before the end of the line. | | L2612(E) | | +| `UnexpectedNameForType` | The '{0}' resource can only be used once per configuration, and therefore cannot have a name. Remove '{1}', and then run the script again. | L3830(E) L3881(E) | | | +| `UnexpectedToken` | Unexpected token '{0}' in expression or statement. | L1362(E) L1588(E) L1672(E) | | | +| `UnexpectedTokenInDynamicKeyword` | There is no whitespace between the name and the scriptblock in dynamic keyword '{0}' body statement. | L3950(E) | | | +| `UnexpectedUnaryOperator` | Internal error - unexpected unary operator {0}. | L5022(E) | | | +| `UnrecognizedToken` | Unrecognized token in source text. | | L4673(E) | | +| `UnrecoverableParserError` | Unrecoverable error in PowerShell. | L108(?) L157(?) | | | +| `UnsupportedReservedKeyword` | The '{0}' keyword is not supported in this version of the language. | L3756(E) | | | +| `UnsupportedReservedProperty` | The '{0}' property is not supported in this version of the language. | L3766(E) | | | +| `UsingMustBeAtStartOfScript` | A 'using' statement must appear before any other statements in a script. | L2140(E) | | | +| `VerboseStream` | verbose stream | L6249(?) | | | +| `WDACParserClassKeywordLogMessage` | The Class keyword will not be allowed in Constrained Language mode for untrusted script. | L4241(E) | | | +| `WDACParserClassKeywordLogTitle` | Parser Class Keyword | L4240(E) | | | +| `WDACParserConfigKeywordLogMessage` | The Configuration keyword will not be allowed in Constrained Language mode for untrusted script. | L3002(E) | | | +| `WDACParserConfigKeywordLogTitle` | Parser Configuration Keyword | L3001(E) | | | +| `WarningStream` | warning stream | L6248(?) | | | +| `WhitespaceBeforeHereStringFooter` | White space is not allowed before the string terminator. | | L2735(I) L2847(I) | | + +Rows: **157** (`GENERATED`). + +## 4. Keys used only elsewhere (semantic checks, type building, runtime) + +| Resource key | Message template | Use count | Files (first three) | +|---|---|---:|---| +| `AddHashTableToNonHashTable` | A hash table can only be added to another hash table. | 1 | `engine/runtime/Binding/Binders.cs` | +| `AmbiguousConversion` | There are valid conversions from {0} to {1}. | 1 | `engine/ComInterop/Errors.cs` | +| `AmbiguousPath` | The path cannot be processed because it resolved to more than one file; only one file at a time can be processed. | 4 | `engine/Modules/ModuleCmdletBase.cs`, `engine/Modules/TestModuleManifestCommand.cs`, `engine/remoting/commands/TestPSSessionConfigurationFile.cs` +1 | +| `AmbiguousTypeReference` | Type name '{0}' is ambiguous, it could be '{1}' or '{2}'. | 3 | `engine/parser/SymbolResolver.cs`, `engine/parser/TypeResolver.cs` | +| `ArrayMultiplyToolongInDataSection` | Array multiplication resulting in more than {0} elements is not allowed in restricted language mode or a Data section. | 1 | `engine/runtime/Operations/ArrayOps.cs` | +| `ArrayReferenceNotSupportedInDataSection` | Array references are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `ArraySliceAssignmentFailed` | Array assignment to [{0}] failed because assignment to slices is not supported. | 1 | `engine/runtime/Binding/Binders.cs` | +| `AsOperatorRequiresType` | The right operand of '-as' must be a type. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `AssignmentStatementNotSupportedInDataSection` | Assignment statements are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `AssignmentStatementToAutomaticNotSupported` | Cannot assign automatic variable '{0}' with type '{1}' | 2 | `engine/parser/SemanticChecks.cs` | +| `AstIsReused` | An AST cannot be used as the child of more than one AST. To use this AST in another AST, call the Copy() method and use its result. | 1 | `engine/parser/ast.cs` | +| `AttributeNotAllowedOnDeclaration` | Attribute '{0}' is not valid on this declaration. It is valid on '{1}' declarations only. | 2 | `engine/parser/SemanticChecks.cs` | +| `AttributeNotSupportedInDataSection` | Attributes are not allowed in restricted language mode or a Data section. | 4 | `engine/parser/SemanticChecks.cs` | +| `BackupParserMessage` | An error '{0}' occurred while processing this script. Text describing this error could not be loaded. | 1 | `engine/lang/parserutils.cs` | +| `BackupParserMessageWithException` | An error '{0}' occurred while processing this script. Text describing this error could not be loaded due to error '{1}'. | 1 | `engine/lang/parserutils.cs` | +| `BadEnumeration` | An error occurred while enumerating through a collection: {0}. | 5 | `engine/lang/parserutils.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `BadExpression` | The expression after '{0}' in a pipeline element produced an object that was not valid. It must result in a command name, a script block, or a CommandInfo object. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `BadOperatorArgument` | The argument to operator '{0}' is not valid: {1}. | 2 | `engine/InternalCommands.cs`, `engine/lang/parserutils.cs` | +| `BadReplaceArgument` | The {0} operator allows only two elements to follow it, not {1}. | 1 | `engine/lang/parserutils.cs` | +| `BaseClassNoDefaultCtor` | Base class '{0}' does not contain a parameterless constructor. | 2 | `engine/parser/PSType.cs` | +| `BreakHelpMessage` | Do not continue processing, throw the exception instead. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `BreakLabel` | &Break | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `COMException` | An unhandled COM interop exception occurred: {0} | 0 | | +| `CannotCall` | Cannot perform call. | 1 | `engine/ComInterop/ComInvokeAction.cs` | +| `CannotConvertValue` | Cannot convert the value to type "{0}". | 4 | `engine/parser/PSType.cs` | +| `CannotCreateTypeConstrainedLanguage` | Cannot create type. Only core types are supported in this language mode. | 1 | `engine/runtime/Binding/Binders.cs` | +| `CannotIndex` | Unable to index into an object of type "{0}". | 2 | `engine/runtime/Binding/Binders.cs`, `engine/runtime/Operations/ArrayOps.cs` | +| `CannotIndexWithByRefLikeReturnType` | Unable to index into an object of type "{0}" with the ByRef-like return type "{1}". ByRef-like types are not supported in PowerShell. | 4 | `engine/runtime/Binding/Binders.cs` | +| `CannotInvoke` | The method cannot be invoked. | 1 | `engine/runtime/Binding/Binders.cs` | +| `CannotLoadPowerShellDataFile` | Failed to load the PowerShell data file '{0}' with the following error: {1} | 2 | `utils/PsUtils.cs` | +| `CannotResolvePowerShellDataFilePath` | Cannot resolve the path '{0}' to a single .psd1 file. | 1 | `utils/PsUtils.cs` | +| `CannotRetrieveTypeInformation` | Cannot retrieve type information. | 1 | `engine/ComInterop/Errors.cs` | +| `CantActivateDocumentInPipeline` | Cannot run a document in the middle of a pipeline: {0}. | 1 | `engine/NativeCommandProcessor.cs` | +| `CantActivateDocumentInPowerShellCore` | Cannot run a document in PowerShell: {0}. | 1 | `engine/NativeCommandProcessor.cs` | +| `CantInvokeCallOperatorAcrossLanguageBoundaries` | Cannot use '&' or '.' operators to invoke a module scope command across language boundaries. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `CantInvokeInBinaryModule` | Cannot use '&' to invoke in the context of binary module '{0}'. Specify a non-binary module after the '&' and try the operation again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `CantInvokeInNonImportedModule` | Cannot use '&' to invoke in the context of module '{0}' because it is not imported. Import the module '{0}' and try the operation again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `CimDeserializationError` | Cim deserializer threw an error when deserializing file {0}. | 2 | `DscSupport/CimDSCParser.cs` | +| `ClassNameNotSameAsDefiningFile` | The class name '{0}' does not match '{1}', the name of the file in which it is defined. Rename either the file name to match the class name or vice versa | 1 | `DscSupport/CimDSCParser.cs` | +| `CmdletNotInAllowedListForDataSection` | The command '{0}' is not allowed in restricted language mode or a Data section. | 4 | `engine/parser/SemanticChecks.cs` | +| `ConfigurationInvalidPropertyName` | An attribute name for resource '{0}' was found that is not valid. An attribute name must be a simple string, and cannot contain variables or expressions. Replace '{1}' with a simple string. | 2 | `engine/parser/SemanticChecks.cs` | +| `ConfigurationNotSupportedInPowerShellCore` | Configuration keyword is not supported in PowerShell 6+. | 1 | `DscSupport/CimDSCParser.cs` | +| `ConstructorCantHaveReturnType` | A constructor cannot specify a return type. | 2 | `engine/parser/SemanticChecks.cs` | +| `ContinueHelpMessage` | Report the error then continue with the next script statement. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ContinueLabel` | &Continue | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ControlLeavingFinally` | Flow of control cannot leave a Finally block. | 2 | `engine/parser/SemanticChecks.cs` | +| `ConvertCimPropertyToObjectPropertyFailed` | Failed to convert the value of CimProperty {0} to the property value of class {1}. | 1 | `DscSupport/CimDSCParser.cs` | +| `CouldNotGetDispId` | Could not get dispatch ID for {0} (error: {1}). | 1 | `engine/ComInterop/Errors.cs` | +| `CustomAttributeTypeNotFound` | Cannot find the type for custom attribute '{0}'. Make sure that the assembly that contains this type is loaded. | 3 | `engine/parser/Compiler.cs`, `engine/parser/SymbolResolver.cs` | +| `CycleInEnumInitializers` | Cannot define enum because of a cycle in the initialization expressions. | 2 | `engine/parser/PSType.cs` | +| `DataSectionAllowedCommandDisallowed` | Cannot define data section. Definition of additional supported commands is not supported in this language mode. | 2 | `engine/parser/SemanticChecks.cs` | +| `DataSectionStatementNotSupportedInDataSection` | The data statement is not allowed in restricted language mode or another Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `DebugModeShouldHaveOneValue` | DebugMode should only have one value. | 1 | `DscSupport/CimDSCParser.cs` | +| `DefaultAllowedVariablesInDataSection` | $PSCulture, $PSUICulture, $true, $false, $null | 2 | `engine/MshCommandRuntime.cs`, `engine/parser/SemanticChecks.cs` | +| `DisabledRefreshModeNotValidForPartialConfig` | PartialConfiguration '{0}' has a Refresh Mode set to Disabled which is not a valid mode for Partial Configurations. Use Pull or Push refresh mode. | 1 | `DscSupport/CimDSCParser.cs` | +| `DispBadParamCount` | Cannot find an overload for "{0}" and the argument count: "{1}" | 1 | `engine/ComInterop/Errors.cs` | +| `DispMemberNotFound` | Error while invoking {0}. Could not find member. | 1 | `engine/ComInterop/Errors.cs` | +| `DispNoNamedArgs` | Error while invoking {0}. Named arguments are not supported. | 1 | `engine/ComInterop/Errors.cs` | +| `DispOverflow` | Error while invoking {0}. Overflow detected. | 1 | `engine/ComInterop/Errors.cs` | +| `DispParamNotOptional` | Error while invoking {0}. A required parameter was omitted. | 1 | `engine/ComInterop/Errors.cs` | +| `DispTypeMismatch` | Exception setting "{0}": Cannot convert the "{1}" value of type "{2}" to type "{3}". | 1 | `engine/ComInterop/Errors.cs` | +| `DoWhileStatementNotSupportedInDataSection` | The Do and While statements are not allowed in restricted language mode or a Data section. | 4 | `engine/parser/SemanticChecks.cs` | +| `DotSourcingNotSupportedInDataSection` | Dot sourcing is not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `DscResourceInvalidGetMethod` | Invalid Get method. Get method must return [{0}] and accepts no parameters. | 4 | `engine/parser/SemanticChecks.cs` | +| `DscResourceInvalidKeyProperty` | Invalid key property. The key property must be of [string], signed/unsigned integer, or Enum types. | 2 | `engine/parser/SemanticChecks.cs` | +| `DscResourceMissingDefaultConstructor` | The DSC resource '{0}' has no default constructor. | 3 | `DscSupport/CimDSCParser.cs`, `engine/parser/SemanticChecks.cs` | +| `DscResourceMissingGetMethod` | The DSC resource '{0}' is missing a Get method that returns [{0}] and accepts no parameters. | 2 | `engine/parser/SemanticChecks.cs` | +| `DscResourceMissingKeyProperty` | The DSC resource '{0}' must have at least one key property (using the syntax [DscProperty(Key)].) | 2 | `engine/parser/SemanticChecks.cs` | +| `DscResourceMissingSetMethod` | The DSC resource '{0}' is missing a Set method that returns [void] and accepts no parameters. | 2 | `engine/parser/SemanticChecks.cs` | +| `DscResourceMissingTestMethod` | The DSC resource '{0}' is missing a Test method that returns [bool] and accepts no parameters. | 2 | `engine/parser/SemanticChecks.cs` | +| `DscResourcesNotFoundDuringParsing` | Unable to load resource '{0}': Resource not found. | 1 | `DscSupport/CimDSCParser.cs` | +| `DuplicateCimClassDefinition` | A second CIM class definition for '{0}' was found while processing the schema file '{1}'. This class was already defined in the file(s) '{2}'. Remove the redundant definition, and then try again. | 1 | `DscSupport/CimDSCParser.cs` | +| `DuplicateFormalParameter` | Duplicate parameter ${0} in parameter list. | 2 | `engine/parser/SemanticChecks.cs` | +| `DuplicateKeyInHashLiteral` | Duplicate keys '{0}' are not allowed in hash literals. | 3 | `engine/parser/SemanticChecks.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `DuplicateKeywordDefinition` | Resource name '{0}' is already being used by another Resource or Configuration. | 1 | `DscSupport/CimDSCParser.cs` | +| `DuplicatePropertyInInstanceDefinition` | Duplicate property assignments are not allowed in an instance definition. | 2 | `engine/parser/SemanticChecks.cs` | +| `DuplicateResourceIdInNodeStatement` | A duplicate resource identifier '{0}' was found while processing the specification for node '{1}'. Change the name of this resource so that it is unique within the node specification. | 1 | `DscSupport/CimDSCParser.cs` | +| `DynamicKeywordSemanticCheckException` | Exception occurred when performing semantic check for dynamic keyword '{0}' with details '{1}'. | 2 | `engine/parser/SemanticChecks.cs` | +| `EmptyCatchNotLast` | Catch block must be the last catch block. | 2 | `engine/parser/SemanticChecks.cs` | +| `EmptyExpressionRequiresANonDefaultMode` | If the expression passed to the Where() operator is null then you must specify a non-Default value for the selection mode argument. Please change the value of the mode argument to a value other than Default and try running your script again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `EmptyFunctionNameInFunctionDefinitionDictionary` | The key property for an entry in the dictionary of functions to define cannot be empty because the key property is used as the function name. Specify a non-empty string as the value of the key property, and then try the operation again. | 2 | `engine/lang/scriptblock.cs`, `engine/runtime/CompiledScriptBlock.cs` | +| `EnumeratorValueMustBeConstant` | Enumerator value must be a constant value. | 2 | `engine/parser/PSType.cs` | +| `EnumeratorValueOutOfBounds` | Enumerator value is either too large or too small for {0}. | 4 | `engine/parser/PSType.cs` | +| `ErrorCollection` | Parser errors were detected. | 0 | | +| `ExceptionActionPromptCaption` | Action to take for this exception: | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ExceptionParsingMOFFile` | Exception parsing MOF file '{0}':{1}. | 1 | `DscSupport/CimDSCParser.cs` | +| `ExceptionTypeAlreadyCaught` | Exception type {0} is already handled by a previous handler. | 2 | `engine/parser/SemanticChecks.cs` | +| `ExpandableStringNotSupportedInDataSection` | Expandable strings are not allowed in restricted language mode or a Data section. | 1 | `engine/parser/SemanticChecks.cs` | +| `ExpectArrayTypeOfPropertyInPSClass` | Property {0} of PowerShell class {1} is not declared as array type, but defined in its configuration instance as instance array type. | 1 | `DscSupport/CimDSCParser.cs` | +| `FailToParseModuleScriptFile` | Failed to parse module script file '{0}' with error '{1}'. | 1 | `DscSupport/CimDSCParser.cs` | +| `FileNotFound` | No files matching '{0}' were found. | 2 | `DscSupport/CimDSCParser.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `FileOpenError` | The current provider ({0}) cannot open a file. | 4 | `engine/Modules/ModuleCmdletBase.cs`, `engine/Modules/TestModuleManifestCommand.cs`, `engine/remoting/commands/TestPSSessionConfigurationFile.cs` +1 | +| `FlowControlStatementNotSupportedInDataSection` | Flow control statements such as Break, Continue, Return, Exit, and Throw are not allowed in restricted language mode or a Data section. | 10 | `engine/parser/SemanticChecks.cs` | +| `ForEachBadGenericConversionTypeSpecified` | The generic collection type [{0}] passed to ForEach() has too many type arguments. Please change the specified type to be a generic collection with only one type argument then try running your script again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ForEachNonexistentMemberReference` | The ForEach() operator could not find a member '{0}' on the target object. Please verify that the named member exists and then try running your script again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ForEachNotSupportCleanBlock` | Script block with a 'clean' block is not supported by the 'ForEach' method. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ForEachTypeConversionFailed` | Unable to convert input to the target type [{0}] passed to the ForEach() operator. Please check the specified type and try running your script again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ForWhileStatementNotSupportedInDataSection` | For and While statements are not allowed in restricted language mode or a Data section. | 4 | `engine/parser/SemanticChecks.cs` | +| `ForeachStatementNotSupportedInDataSection` | Foreach statements are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `FormatError` | Error formatting a string: {0}. | 1 | `engine/runtime/Operations/StringOps.cs` | +| `FunctionDeclarationNotSupportedInDataSection` | Function declarations are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `GetBadlyFormedExclusiveResourceId` | The format of the resource reference '{0}' in the exclusive list for resource '{1}' is not valid. An exclusive resource name should be in the format '\', with no spaces. | 1 | `DscSupport/CimDSCParser.cs` | +| `GetBadlyFormedRequiredResourceId` | The format of the resource reference '{0}' in the Requires list for resource '{1}' is not valid. A required resource name should be in the format '[]', with alphanumeric characters, spaces, '_', '-', '.' and '\'. | 1 | `DscSupport/CimDSCParser.cs` | +| `GetIDsOfNamesInvalid` | IDispatch::GetIDsOfNames behaved unexpectedly for {0}. | 1 | `engine/ComInterop/Errors.cs` | +| `GetPullModeNeedConfigurationSource` | The PartialConfiguration '{0}' is set to pull mode which requires a ConfigurationSource property. | 1 | `DscSupport/CimDSCParser.cs` | +| `GetSteppablePipelineFromWrongThread` | There is no Runspace available to get and run the SteppablePipeline in this thread. You can provide one in the DefaultRunspace property of the System.Management.Automation.Runspaces.Runspace type. The script block you attempted to get SteppablePipeline from was: {0} | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `HashtableToObjectConversionNotSupportedInDataSection` | Cannot convert hashtable to an object of the following type: {0}. Hashtable-to-Object conversion is not supported in restricted language mode or a Data section. | 1 | `engine/LanguagePrimitives.cs` | +| `ImportDscResourceInsideNode` | Import-DscResource cannot be specified inside of Node context | 1 | `DscSupport/CimDSCParser.cs` | +| `ImportDscResourceMultipleModulesNotSupportedWithName` | Import-DscResource dynamic keyword supports only one module when Name parameter is specified. | 1 | `DscSupport/CimDSCParser.cs` | +| `ImportDscResourceNeedParams` | The syntax of the Import-DscResource dynamic keyword is: Import-DscResource [-Name ] [-ModuleName ] [-ModuleVersion ]. Name : Names of one or more resources to import. ModuleName : Module names or ModuleSpecification objects of one or more modules to import. ModuleVersion : Version of module to import. If used, ModuleName must represent only one module by name. | 5 | `DscSupport/CimDSCParser.cs` | +| `ImportDscResourcePositionalParamsNotSupported` | Positional parameters are not supported for the Import-DscResource dynamic keyword. The syntax of Import-DscResource dynamic keyword is: "Import-DscResource [-Name ] [-ModuleName ] [-ModuleVersion ] | 1 | `DscSupport/CimDSCParser.cs` | +| `IndexOutOfRange` | Array assignment failed because index '{0}' was out of range. | 0 | | +| `InstantiatePSClassObjectFailed` | Failed to create an object of PowerShell class {0}. | 1 | `DscSupport/CimDSCParser.cs` | +| `InterfaceNameExpected` | '{0}': Interface name expected. | 2 | `engine/parser/PSType.cs` | +| `InvalidComObjectException` | A COM object was accessed after it was already released: {0} | 1 | `engine/CommandProcessorBase.cs` | +| `InvalidEndOfLine` | The token '{0}' is not a valid statement separator in this version. | 2 | `engine/parser/SemanticChecks.cs` | +| `InvalidFilenameOption` | The file name argument to -file in the switch statement is not valid. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `InvalidHashtable` | The hashtable supplied to the Desired State Configuration resource {0} is not valid. The key or value cannot be null or empty. | 2 | `DscSupport/CimDSCParser.cs` | +| `InvalidInstanceProperty` | The member '{0}' is not valid. Valid members are '{1}'. | 2 | `engine/parser/SemanticChecks.cs` | +| `InvalidLeftHandSide` | The assignment expression is not valid. The input to an assignment operator must be an object that is able to accept assignments, such as a variable or a property. | 2 | `engine/parser/SemanticChecks.cs` | +| `InvalidLocalConfigurationManagerProperty` | '{0}' cannot be specified in LocalConfigurationManager resource. Please switch to Settings instead or use only following values: {1}. | 1 | `DscSupport/CimDSCParser.cs` | +| `InvalidNamespaceValue` | The specified namespace in the 'using' statement contains invalid characters. | 2 | `engine/parser/SemanticChecks.cs` | +| `InvalidNullKey` | A null key is not allowed in a hash literal. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `InvalidParameter` | Parameter '{0}' is not valid | 0 | | +| `InvalidPassword` | The username supplied to the Desired State Configuration resource {0} is not valid. The username cannot be null or empty. | 2 | `DscSupport/CimDSCParser.cs` | +| `InvalidPowerShellDataFile` | The PowerShell data file '{0}' is invalid since it cannot be evaluated into a Hashtable object. | 3 | `utils/PsUtils.cs` | +| `InvalidRegularExpression` | The regular expression pattern {0} is not valid. | 2 | `engine/lang/parserutils.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `InvalidScriptBlockInDataSection` | A Begin statement block, Process statement block, or parameter statement is not allowed in a Data section. | 2 | `engine/runtime/CompiledScriptBlock.cs` | +| `InvalidSplitOptionCombination` | The combination of options with the -split operator is not valid. | 1 | `engine/lang/parserutils.cs` | +| `InvalidSplitOptionWithPredicate` | Options are not allowed on the -split operator with a predicate. | 1 | `engine/lang/parserutils.cs` | +| `InvalidUserName` | The username supplied to the Desired State Configuration resource {0} is not valid. The username cannot be null or empty. | 2 | `DscSupport/CimDSCParser.cs` | +| `InvalidUsingExpression` | Expression is not allowed in a Using expression. | 2 | `engine/parser/SemanticChecks.cs` | +| `InvalidValueForProperty` | '{0}' is not a valid value for property '{1}' on class '{2}'. Please change the value to one of the following strings: {3}. | 1 | `DscSupport/CimDSCParser.cs` | +| `InvokeMethodConstrainedLanguage` | Cannot invoke method. Method invocation is supported only on core types in this language mode. | 3 | `engine/InternalCommands.cs`, `engine/runtime/Binding/Binders.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `InvokeMethodOnNull` | You cannot call a method on a null-valued expression. | 4 | `engine/InternalCommands.cs`, `engine/lang/parserutils.cs`, `engine/runtime/Binding/Binders.cs` | +| `InvokingCleanBlockNotSupported` | Directly invoking the 'clean' block of a script block is not supported. | 1 | `engine/runtime/CompiledScriptBlock.cs` | +| `IsOperatorRequiresType` | The right operand of '-is' must be a type. | 3 | `engine/lang/parserutils.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `KeywordParameterReservedForFutureUse` | The {0} '-{1}' parameter is reserved for future use. | 6 | `engine/parser/SemanticChecks.cs` | +| `LabelNotFound` | Label '{0}' not found inside the method. | 2 | `engine/parser/SemanticChecks.cs` | +| `MemberAlreadyDefined` | The member '{0}' is already defined. | 10 | `engine/parser/PSType.cs` | +| `MetaConfigurationUsedInRegularConfig` | Meta Configuration resource {0} is not allowed in the regular configuration. Use meta configuration resources in a configuration with [DscLocalConfigurationManager()] attribute. | 2 | `engine/parser/SemanticChecks.cs` | +| `MethodCallNotSupportedInDataSection` | Method calls are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `MethodHasCodePathNotReturn` | Not all code path returns value within method. | 2 | `engine/parser/SemanticChecks.cs` | +| `MethodNotFound` | Method invocation failed because [{0}] does not contain a method named '{1}'. | 8 | `engine/runtime/Binding/Binders.cs` | +| `MissingThis` | Cannot assign property, use '{0}{1}'. | 2 | `engine/parser/SymbolResolver.cs` | +| `MissingTryStatement` | Incomplete 'try' statement. A try statement requires a body. | 0 | | +| `MissingTypeInStaticPropertyAssignment` | Cannot assign property, use '{0}{1}'. | 2 | `engine/parser/SymbolResolver.cs` | +| `MissingValueForMandatoryProperty` | Resource '{0}' requires that a value of type '{1}' be provided for property '{2}'. | 2 | `DscSupport/CimDSCParser.cs` | +| `ModuleNotFoundDuringParse` | Could not find the module '{0}'. | 3 | `DscSupport/CimDSCParser.cs`, `engine/parser/SymbolResolver.cs` | +| `MultipleModuleEntriesFoundDuringParse` | Multiple versions of the module '{0}' were found. You can run 'Get-Module -ListAvailable -FullyQualifiedName {0}' to see available versions on the system, and then use the fully qualified name '@{{ModuleName="{0}"; RequiredVersion="Version"}}'. | 1 | `DscSupport/CimDSCParser.cs` | +| `MultipleTypeConstraintsOnMethodParam` | Multiple type constraints are not allowed on a method parameter. | 2 | `engine/parser/SemanticChecks.cs` | +| `NamedBlockNotAllowedInMethod` | A named block is not allowed in a class method. | 2 | `engine/parser/SemanticChecks.cs` | +| `NeedMultidimensionalIndex` | You cannot index into a {0} dimensional array with index [{1}]. | 4 | `engine/runtime/Binding/Binders.cs`, `engine/runtime/Operations/ArrayOps.cs` | +| `NonExistingVariableReference` | [ref] cannot be applied to a variable that does not exist. | 1 | `engine/runtime/Operations/VariableOps.cs` | +| `NonStaticMemberAccessInStaticMember` | Cannot access the non-static member '{0}' in a static method or initializer of a static property. | 2 | `engine/parser/SemanticChecks.cs` | +| `NonVoidMethodMissingReturnValue` | Invalid return statement within non-void method. | 2 | `engine/parser/SemanticChecks.cs` | +| `NotADefinedOperationForType` | The operation '[{0}] {1} [{2}]' is not defined. | 1 | `engine/lang/parserutils.cs` | +| `NullArray` | Cannot index into a null array. | 2 | `engine/runtime/Binding/Binders.cs` | +| `NullArrayIndex` | Index operation failed; the array index evaluated to null. | 2 | `engine/runtime/Binding/Binders.cs` | +| `NullEntryInVariablesDefinitionList` | A null entry was found in the list of variable entries to create in the script block scope. Remove the entry at index {0}, or replace it with a non-null entry, and then try again. | 1 | `engine/runtime/CompiledScriptBlock.cs` | +| `NullFunctionBodyInFunctionDefinitionDictionary` | The script block that defines function '{0}' cannot be null or empty. Provide a non-empty script block in the function definition dictionary, and then try the operation again. | 1 | `engine/runtime/CompiledScriptBlock.cs` | +| `NumberToReturnMustBeGreaterThanZero` | The 'numberToReturn' value provided to the third argument of the Where() operator must be greater than zero. Please correct the argument's value and try running your script again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `OnlyOneParameterListAllowed` | The param statement cannot be used if arguments were specified in the function declaration. | 2 | `engine/parser/SemanticChecks.cs` | +| `OperatorFailed` | The '{0}' operator failed: {1}. | 1 | `engine/InternalCommands.cs` | +| `OperatorNotSupportedInDataSection` | The '{0}' operator is not allowed in restricted language mode or a Data section. | 6 | `engine/parser/SemanticChecks.cs` | +| `OperatorRequiresNumber` | The '{0}' operator works only on numbers. The operand is a '{1}'. | 1 | `engine/runtime/Binding/Binders.cs` | +| `OperatorRequiresVariableOrProperty` | The '{0}' operator works only on variables or on properties. | 2 | `engine/parser/SemanticChecks.cs` | +| `OrderedAttributeOnlyOnHashLiteralNode` | The {0} attribute can be specified only on a hash literal node. | 4 | `engine/parser/SemanticChecks.cs` | +| `ParallelAndSequenceBlockNotSupportedInDataSection` | Parallel and sequence blocks are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `ParamBlockNotAllowedInMethod` | A param block is not allowed in a class method. | 2 | `engine/parser/SemanticChecks.cs` | +| `ParameterAttributeArgumentNeedsToBeConstant` | Attribute argument must be a constant. | 2 | `engine/parser/SemanticChecks.cs` | +| `ParameterAttributeArgumentNeedsToBeConstantOrScriptBlock` | Attribute argument must be a constant or a script block. | 2 | `engine/parser/SemanticChecks.cs` | +| `ParameterDeclarationNotSupportedInDataSection` | Parameter declarations are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `ParameterizedPropertyAssignmentFailed` | Assignment failed because [{0}] does not contain a property '{1}()' that can be set. | 2 | `engine/lang/parserutils.cs` | +| `ProgramFailedToExecute` | Program '{0}' failed to run: {1}{2}. | 2 | `engine/NativeCommandProcessor.cs` | +| `PropertyInGenericType` | '{0}' is defined in a generic type. | 3 | `engine/CoreAdapter.cs` | +| `PropertyIsReadOnly` | '{0}' is a ReadOnly property. | 4 | `engine/CoreAdapter.cs`, `engine/runtime/Binding/Binders.cs` | +| `PropertyNotDeclaredInPSClass` | Property {0} is not declared in PowerShell class {1}, but defined in its configuration instance. | 1 | `DscSupport/CimDSCParser.cs` | +| `PropertyNotFound` | The property '{0}' cannot be found on this object. Verify that the property exists and can be set. | 3 | `engine/runtime/Binding/Binders.cs` | +| `PropertyNotFoundForAttribute` | Property '{0}' cannot be found for attribute '{1}'. Specify one of the following properties: {2}. | 2 | `engine/parser/SemanticChecks.cs` | +| `PropertyNotFoundForType` | Property '{0}' cannot be found for type '{1}'. | 6 | `engine/parser/Compiler.cs`, `engine/runtime/Binding/Binders.cs` | +| `PropertyNotFoundStrict` | The property '{0}' cannot be found on this object. Verify that the property exists. | 5 | `engine/runtime/Binding/Binders.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `PropertyReferenceNotSupportedInDataSection` | Property references are not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `PropertySetConstrainedLanguage` | Cannot set property. Property setting is supported only on core types in this language mode. | 3 | `engine/runtime/Binding/Binders.cs`, `engine/runtime/Operations/MiscOps.cs` | +| `PsDscMissingSchemaStore` | Unable to find DSC schema store at "{0}". Please ensure PSDesiredStateConfiguration v3 module is installed. | 0 | | +| `PsDscRunAsCredentialMergeErrorForCompositeResources` | Conflict in using PsDscRunAsCredential for Resource {0} because it already specifies PsDscRunAsCredential value. We can only use one PsDscRunAsCredential for the composite resource. | 1 | `DscSupport/CimDSCParser.cs` | +| `RedirectionFailed` | Redirection to '{0}' failed: {1} | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `RedirectionNotSupportedInDataSection` | Redirection is not allowed in restricted language mode or a Data section. | 4 | `engine/parser/SemanticChecks.cs` | +| `RedirectionStreamCanOnlyMergeToOutputStream` | Redirection only allows another stream to be merged with the output stream. Please correct the redirection operation to merge into the output stream then try running your script again. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `ReferenceNeedsToBeByItselfInTypeConstraint` | Cannot use [ref] with other types in a type constraint. | 2 | `engine/parser/SemanticChecks.cs` | +| `ReferenceNeedsToBeByItselfInTypeSequence` | Cannot have two occurrences of [ref] in a type sequence. | 2 | `engine/parser/SemanticChecks.cs` | +| `ReferenceNeedsToBeLastTypeInTypeConversion` | [ref] can only be the final type in type conversion sequence. | 2 | `engine/parser/SemanticChecks.cs` | +| `RegularResourceUsedInMetaConfig` | Regular DSC resource {0} is not allowed in the meta configuration. | 2 | `engine/parser/SemanticChecks.cs` | +| `ResourceNotDefined` | Undefined DSC resource '{0}'. Use Import-DSCResource to import the resource. | 2 | `engine/parser/SemanticChecks.cs` | +| `SchemaFileNotFound` | The schema file {0} was not found. Verify that any modules specified in a configuration statement contain a schema.mof file, and then try running the script again. | 2 | `DscSupport/CimDSCParser.cs` | +| `ScriptBlockDelegateInvokedFromWrongThread` | There is no Runspace available to run scripts in this thread. You can provide one in the DefaultRunspace property of the System.Management.Automation.Runspaces.Runspace type. The script block you attempted to invoke was: {0} | 1 | `engine/lang/scriptblock.cs` | +| `ScriptBlockNotSupportedInDataSection` | Script block literals are not allowed in restricted language mode or a Data section. | 4 | `engine/parser/SemanticChecks.cs` | +| `ScriptContainedMaliciousContent` | This script contains malicious content and has been blocked by your antivirus software. | 2 | `engine/runtime/CompiledScriptBlock.cs`, `security/SecuritySupport.cs` | +| `ScriptHasAdminBlockedContent` | This script contains content that has been flagged as suspicious through a policy setting and has been blocked with error code {0}. Contact your administrator for more information. | 1 | `engine/runtime/CompiledScriptBlock.cs` | +| `ScriptsNotAllowed` | The syntax is not supported by this runspace. This can occur if the runspace is in no-language mode. | 1 | `engine/hostifaces/Command.cs` | +| `SealedBaseClass` | Cannot inherit from sealed class '{0}'. | 2 | `engine/parser/PSType.cs` | +| `SetComObjectDataFailed` | Marshal.SetComObjectData failed. | 1 | `engine/ComInterop/Errors.cs` | +| `SilentlyContinueHelpMessage` | Do not report this error, just continue with the next script statement. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `SilentlyContinueLabel` | S&ilently Continue | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `SplattingNotPermitted` | The splatting operator '@' cannot be used to reference variables in an expression. '@{0}' can be used only as an argument to a command. To reference variables in an expression use '${0}'. | 2 | `engine/parser/SemanticChecks.cs` | +| `SplattingNotPermittedInArgumentList` | Splatted variables like '@{0}' cannot be part of a comma-separated list of arguments. | 2 | `engine/parser/SemanticChecks.cs` | +| `StaticConstructorCantHaveParameters` | A static constructor cannot have any parameters. | 2 | `engine/parser/PSType.cs` | +| `StrictModeFunctionCallWithParens` | The function or command was called as if it were a method. Parameters should be separated by spaces. For information about parameters, see the about_Parameters Help topic. | 1 | `engine/parser/Compiler.cs` | +| `StringMultiplyToolongInDataSection` | String multiplication results with more than "{0}" characters are not allowed in restricted language mode or a Data section. | 1 | `engine/runtime/Operations/StringOps.cs` | +| `SubtypeArray` | Invalid base type '{0}'. Base type cannot be an array. | 6 | `engine/parser/PSType.cs` | +| `SubtypeUnclosedGeneric` | Invalid base type '{0}'. Base type cannot be a generic with unspecified parameters. | 2 | `engine/parser/PSType.cs` | +| `SuspendHelpMessage` | Pause the current pipeline and return to the command prompt. Type exit to resume operation when you are done. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `SuspendLabel` | &Suspend | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `SwitchStatementNotSupportedInDataSection` | The switch statement is not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `TextForPositionMessage` | At {0}:{1} char:{2} + {3} | 1 | `engine/parser/Position.cs` | +| `TextForWordLine` | line | 1 | `engine/parser/Position.cs` | +| `ThrottleLimitRequiresParallelFlag` | The ThrottleLimit parameter is only supported on foreach statements that use the Parallel parameter. | 2 | `engine/parser/SemanticChecks.cs` | +| `TooManyErrors` | Not all parse errors were reported. Correct the reported errors and try again. | 1 | `utils/ParserException.cs` | +| `TraceEnteringFunction` | ! CALL function '{0}' | 1 | `engine/debugger/debugger.cs` | +| `TraceEnteringFunctionDefinedInFile` | ! CALL function '{0}' (defined in file '{1}') | 1 | `engine/debugger/debugger.cs` | +| `TraceMethodCall` | ! CALL method '{0}' | 1 | `engine/CoreAdapter.cs` | +| `TraceScriptLineMessage` | {0,4}+ {1} | 1 | `engine/parser/Position.cs` | +| `TraceVariableAssignment` | ! SET ${0} = '{1}'. | 1 | `engine/debugger/debugger.cs` | +| `TrapStatementNotSupportedInDataSection` | The Trap statement is not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `TryStatementNotSupportedInDataSection` | The Try statement is not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `TypeCreationError` | Error during creation of type "{0}". Error message: {1} | 2 | `engine/parser/PSType.cs` | +| `TypeNotAllowedForParameter` | The type '{0}' is not allowed on a parameter. | 2 | `engine/parser/PSType.cs` | +| `TypeNotAllowedForProperty` | The type '{0}' is not allowed on a property. | 2 | `engine/parser/SemanticChecks.cs` | +| `TypeNotAllowedInDataSection` | The type {0} is not allowed in restricted language mode or a Data section. | 2 | `engine/parser/SemanticChecks.cs` | +| `TypeNotFound` | Unable to find type [{0}]. | 14 | `engine/TypeTable.cs`, `engine/parser/PSType.cs` | +| `TypeNotFoundWithMessage` | Unable to find type [{0}]. Details: {1} | 2 | `engine/runtime/Operations/MiscOps.cs` | +| `UnexpectedKeyword` | Unexpected keyword '{0}'. | 2 | `engine/parser/SemanticChecks.cs` | +| `UnexpectedVarEnum` | Unexpected VarEnum {0}. | 1 | `engine/ComInterop/Errors.cs` | +| `UnsupportedHandlerType` | Attempting to pass an event handler of an unsupported type. | 1 | `engine/ComInterop/Errors.cs` | +| `UnsupportedNamedBlockInConfiguration` | Configuration only supports the End block in its body. Begin, Process and DynamicParam blocks are not allowed in a configuration. | 2 | `engine/parser/SemanticChecks.cs` | +| `UnsupportedPropertyTypeOfDSCResourceClass` | The '{0}' property with type '{1}' of DSC resource class '{2}' is not supported. | 2 | `DscSupport/CimDSCParser.cs` | +| `UnsupportedValueForProperty` | At least one of the values '{0}' is not supported or valid for property '{1}' on class '{2}'. Please specify only supported values: {3}. | 1 | `DscSupport/CimDSCParser.cs` | +| `UsingStatementNotSupported` | This syntax of the 'using' statement is not supported. | 2 | `engine/parser/SemanticChecks.cs` | +| `UsingWithoutInvokeCommand` | A Using variable cannot be retrieved. A Using variable can be used only with Invoke-Command, Start-Job, or InlineScript in the script workflow. When it is used with Invoke-Command, the Using variable is valid only if the script block is invoked on a remote computer. | 1 | `engine/runtime/Operations/VariableOps.cs` | +| `ValueNotInRange` | Property '{0}' of Resource '{1}' has value '{2}' which is not between valid range '{3}' and '{4}'. | 1 | `DscSupport/CimDSCParser.cs` | +| `VariableIsUndefined` | The variable '${0}' cannot be retrieved because it has not been set. | 2 | `engine/runtime/Operations/VariableOps.cs` | +| `VariableNotLocal` | Variable is not assigned in the method. | 2 | `engine/parser/SemanticChecks.cs` | +| `VariableReferenceNotSupportedInDataSection` | A variable that cannot be referenced in restricted language mode or a Data section is being referenced. Variables that can be referenced include the following: {0}. | 3 | `engine/MshCommandRuntime.cs`, `engine/parser/SemanticChecks.cs` | +| `VoidMethodHasReturn` | Invalid return statement within void method. | 2 | `engine/parser/SemanticChecks.cs` | +| `VoidTypeConstraintNotAllowed` | [void] cannot be used as a parameter type, or on the left side of an assignment. | 4 | `engine/parser/SemanticChecks.cs` | +| `WDACGetSteppablePipelineLogMessage` | Creating a steppable pipeline from a script block may require evaluating some expressions within the script block. The expression evaluation will silently fail and return 'null' in Constrained Language mode, unless the expression represents a constant value. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `WDACGetSteppablePipelineLogTitle` | Expression Evaluation May Fail | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `WDACParserDSSupportedCommandLogMessage` | The Data Section that includes the SupportedCommand parameter would be disallowed in Constrained Language mode for untrusted script. | 1 | `engine/parser/SemanticChecks.cs` | +| `WDACParserDSSupportedCommandLogTitle` | Parser Data Section SupportedCommand | 1 | `engine/parser/SemanticChecks.cs` | +| `WDACParserForEachOperatorLogMessage` | The ForEach keyword will fail '{0}' iteration item method invocation when run in Constrained Language mode. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `WDACParserForEachOperatorLogTitle` | ForEach Keyword Method Invocation | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `WDACParserModuleScopeCallOperatorLogMessage` | The module scope call operator will be denied in Constrained Language mode. | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `WDACParserModuleScopeCallOperatorLogTitle` | Module Scope Call Operator | 1 | `engine/runtime/Operations/MiscOps.cs` | +| `WildCardModuleNameError` | Wildcard characters are not supported for module names. | 2 | `engine/parser/SymbolResolver.cs` | +| `WorkflowNotSupportedInPowerShellCore` | Workflow is not supported in PowerShell 6+. | 2 | `engine/parser/SemanticChecks.cs` | + +Rows: **255** (`GENERATED`). Keys with use count 0 have no reference found by the dataset scan (dead or referenced reflectively: `UNVERIFIED`). + +## 5. Cross-checks + +`ParserStrings` is not the only resource class a parser error ID can come from: the debug-only check `AssertErrorIdCorrespondsToMsgString` looks the ID up in `ParserStrings`, `DiscoveryExceptions`, `ExtendedTypeSystem`, `MshSnapInCmdletResources` and `ParameterBinderStrings` (`FACT-SRC`, `src/System.Management.Automation/engine/parser/Parser.cs::Parser.AssertErrorIdCorrespondsToMsgString@L8065-L8093`). Only `ParserStrings` is cataloged here. + +Verification: key count taken from `data/resources.json` (`ParserStrings.resx`, key_count 412) and reconciled with the `` element count of the resx parsed with `xml.etree` (`412`); every row has a message from the resx. + diff --git a/docs-out/02-language/GRAMMAR.ebnf.md b/docs-out/02-language/GRAMMAR.ebnf.md new file mode 100644 index 0000000..304b30c --- /dev/null +++ b/docs-out/02-language/GRAMMAR.ebnf.md @@ -0,0 +1,186 @@ +--- +id: LANG-GRAMMAR-001 +title: Grammar (EBNF reconstructed from the parser) +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-GRM-001] +sources: [src/System.Management.Automation/engine/parser/Parser.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/parser-methods.json] +--- + +# Grammar (EBNF reconstructed from the parser) + +> STATIC-ONLY — behavior not run-verified. Every production below was written from the parser code (`FACT-SRC`); the `// G` grammar comments in the source were used only as hints. Where code and comment disagree the code wins and the production is marked **[≠G]** (recorded as contradiction CON-004). Statement terminators and the newline rules are canonical fact CF-GRM-001, stated in §3a. + +## 1. Purpose and scope [SPEC] + +An implementer-oriented grammar of the PowerShell language as accepted by `Parser.cs`. It describes **syntax**, not semantics; operator precedence levels are in [02.2-parser.md](02.2-parser.md) §7.2 (generated) and `OPERATOR-TABLE.md` (planned) (planned T28). Tokenizer-mode dependent lexical rules are in [02.1-tokenizer.md](02.1-tokenizer.md); this grammar uses the token names of [TOKEN-CATALOG.md](TOKEN-CATALOG.md). + +Notation: `::=` defines; `|` alternative; `[ x ]` optional; `{ x }` zero or more; `( … )` grouping; `'text'` a token by its text; `Name` a `TokenKind` (e.g. `Variable`, `Generic`); lower-case-hyphen names are non-terminals. `nl` = zero or more `NewLine` tokens (`SkipNewlines`); `nls` = zero or more `NewLine` or `Semi` tokens (`SkipNewlinesAndSemicolons`). "Mode" says which tokenizer mode is active while the production is parsed (C = Command, E = Expression, T = TypeName, S = Signature). Each production names its parser method; all methods are in `src/System.Management.Automation/engine/parser/Parser.cs` (line ranges in [02.2-parser.md](02.2-parser.md) §7.1). + +## 2. Source map + +| path | type | role | +|---|---|---| +| `src/System.Management.Automation/engine/parser/Parser.cs::Parser@L36-L8191` | class | one `*Rule` method per production below | + +## 3. Script structure + +```ebnf +script-block ::= [ using-statements ] [ param-block ] nls [ script-block-body ] (* ScriptBlockRule; using only at top level *) +using-statements ::= { using-statement } (* UsingStatementsRule *) +script-block-body ::= named-block-list | statement-list-and-closer (* ScriptBlockBodyRule *) +named-block-list ::= { nls block-name statement-block } (* NamedBlockListRule *) +block-name ::= 'begin' | 'process' | 'end' | 'dynamicparam' | 'clean' +param-block ::= nl [ attribute-list ] nl 'param' nl '(' [ parameter-list ] nl ')' (* ParamBlockRule; no '(' => not a param block *) +parameter-list ::= parameter { nl ',' parameter } (* ParameterListRule *) +parameter ::= nl [ attribute-list ] nl ( Variable | SplattedVariable ) [ nl '=' nl expression ] (* ParameterRule, mode E, commas disabled *) +attribute-list ::= { attribute | type-constraint } (* AttributeListRule *) +attribute ::= '[' type-spec '(' attribute-arguments ')' ']' (* AttributeRule *) +type-constraint ::= '[' type-spec ']' +attribute-arguments ::= attribute-argument { nl ',' attribute-argument } (* AttributeArgumentsRule *) +attribute-argument ::= nl ( simple-name '=' nl expression | nl expression | simple-name ) (* named, positional, or name only => true *) +statement-block ::= nl '{' [ statement-list ] nl '}' (* StatementBlockRule *) +statement-list ::= nls { statement nls } (* StatementListRule; stops at ')' or '}' *) +``` + +Notes: a stray token at top level that cannot start a statement is an error and is skipped (`CompleteScriptBlockBody`). `Trap` statements are collected separately from other statements. Duplicate named blocks are an error (`DuplicateScriptCommandClause`) but parsing continues. + +## 3a. Statement terminators and newlines (canonical fact) + +**Statement and terminator rule.** A statement list is parsed as: skip any run of newlines and semicolons; parse a statement; skip any run of newlines and semicolons; stop when the next token is `)` or `}` or no statement can be parsed. A terminator is **not** required between statements that end with a closing `}` (e.g. `if`, loops, `try`, `function`), because the list only skips optional terminators; a pipeline (or any statement parsed by `PipelineChainRule`) ends at newline, `;`, `)`, `}`, `&&`/`||` (chain), `&` (background) or end of input, and any other token after a complete pipeline element is `UnexpectedToken`. Newlines continue a construct only in these places: after `|`, before a `|` that starts the next line, after `&&` and `||`, after each binary or unary operator and after `,`, after `=` in an assignment, inside `( )`, after `?`/`:` of a ternary, between `}` and `elseif`/`else`/`catch`/`finally`, and after a backtick line continuation handled by the tokenizer. + +Evidence: `src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944`, `Parser.PipelineChainRule@L5773-L5988`, `Parser.PipelineRule@L5990-L6160`, `Parser.IfStatementRule@L2400-L2541`, `Parser.CatchBlockRule@L5494-L5591`, `Parser.TryStatementRule@L5593-L5670`. + +## 4. Statements + +```ebnf +statement ::= [ attribute-list ] ( keyword-statement | labeled-statement | pipeline-chain ) (* StatementRule *) +keyword-statement ::= if-statement | function-statement | flow-statement | trap-statement + | try-statement | data-statement | class-statement | enum-statement + | using-statement | block-statement | configuration-statement | dynamic-keyword-statement +labeled-statement ::= [ Label ] ( switch-statement | foreach-statement | for-statement | while-statement | do-statement ) (* LabeledStatementRule; label before other => pipeline *) +if-statement ::= 'if' nl '(' pipeline-chain ')' statement-block + { nl 'elseif' nl '(' pipeline-chain ')' statement-block } [ nl 'else' statement-block ] (* IfStatementRule *) +while-statement ::= 'while' nl '(' nl pipeline-chain nl ')' statement-block (* WhileStatementRule *) +do-statement ::= 'do' statement-block nl ( 'while' | 'until' ) nl '(' pipeline-chain nl ')' (* DoWhileStatementRule *) +for-statement ::= 'for' nl '(' nl [ pipeline-chain ] [ ';' nl [ pipeline-chain ] [ ';' nl [ pipeline-chain ] ] ] nl ')' statement-block (* ForStatementRule; three forms *) +foreach-statement ::= 'foreach' nl { '-parallel' | '-throttlelimit' nl command-argument } nl + '(' nl ( Variable | SplattedVariable ) nl 'in' nl pipeline-chain nl ')' statement-block (* ForeachStatementRule *) +switch-statement ::= 'switch' nl { switch-flag } ( '(' nl pipeline-chain nl ')' | '-file' nl command-argument ) nl switch-body (* SwitchStatementRule *) +switch-flag ::= '-regex' | '-wildcard' | '-exact' | '-casesensitive' | '-parallel' (* matched by prefix, case-insensitive *) +switch-body ::= '{' nls { switch-clause nls } '}' +switch-clause ::= ( 'default' | command-argument-or-primary ) statement-block +try-statement ::= 'try' statement-block ( catch-clause { catch-clause } [ finally-clause ] | finally-clause ) (* TryStatementRule *) +catch-clause ::= nl 'catch' [ type-literal { nl ',' nl type-literal } ] statement-block (* CatchBlockRule *) +finally-clause ::= nl 'finally' statement-block +trap-statement ::= 'trap' nl [ type-literal ] statement-block (* TrapStatementRule *) +data-statement ::= 'data' nl [ simple-name ] [ '-supportedcommand' command-name-list ] statement-block (* DataStatementRule *) +flow-statement ::= ( 'break' | 'continue' ) [ label-expression ] (* BreakStatementRule ContinueStatementRule *) + | ( 'return' | 'exit' | 'throw' ) [ pipeline-chain ] (* ReturnStatementRule ExitStatementRule ThrowStatementRule [≠G: throw uses pipeline-chain, comment says pipeline] *) +label-expression ::= simple-name | unary-expression +block-statement ::= ( 'parallel' | 'sequence' ) statement-block (* BlockStatementRule *) +``` + +Notes (`FACT-SRC`): +- `else`, `elseif`, `catch`, `until` at statement start with earlier errors are eaten; without errors they are treated as command names (`StatementRule`). +- `from`, `define`, `var` at statement start produce `ReservedKeywordNotAllowed`. +- `using` anywhere but the start of the script parses, then reports `UsingMustBeAtStartOfScript`. +- In `if`, the restore point after the last `}` preserves a following newline so hashtable contexts are not swallowed. +- Break/continue accept an optional label expression; `return`/`exit` use `pipeline-chain`. + +## 5. Functions, classes, enums, using + +```ebnf +function-statement ::= ( 'function' | 'filter' | 'workflow' ) nl function-name [ function-params ] statement-block (* FunctionDeclarationRule *) +function-name ::= command-argument (* name token may not be punctuation, string, here-string or variable: MissingNameAfterKeyword *) +function-params ::= nl '(' parameter-list nl ')' (* FunctionParameterDeclarationRule *) +class-statement ::= [ attribute-list ] 'class' nl simple-name [ nl ':' nl type-name { nl ',' nl type-name } ] nl '{' { class-member } '}' (* ClassDefinitionRule; mode S for ':' *) +class-member ::= { attribute | 'hidden' | 'static' } ( property-member | method-member ) (* ClassMemberRule *) +property-member ::= [ type-constraint ] Variable [ '=' expression ] ( NewLine | ';' | lookahead '}' ) +method-member ::= [ type-constraint ] method-name '(' [ parameter-list ] ')' [ ':' 'base' '(' arguments ')' ] '{' script-block '}' (* MethodDeclarationRule; base call only for constructors *) +enum-statement ::= 'enum' nl simple-name [ ':' underlying-type ] nl '{' { enum-member } '}' (* EnumDefinitionRule *) +enum-member ::= simple-name [ '=' expression ] ( NewLine | ';' | lookahead '}' ) (* EnumMemberRule *) +using-statement ::= 'using' ( 'namespace' name [ '=' name ] | 'module' ( name | hashtable ) [ '=' ( name | hashtable ) ] + | 'type' name '=' name | 'assembly' name | 'command' name '=' name ) term (* UsingStatementRule; item must be a constant string, alias rules per directive *) +``` + +Notes: valid enum underlying types are Byte, Int16, Int32, Int64, SByte, UInt16, UInt32, UInt64 (`ValidUnderlyingTypeCodes`, `src/System.Management.Automation/engine/parser/Parser.cs::Parser.EnumDefinitionRule@L4701-L4820`). Class members: duplicate `hidden`/`static` is `DuplicateQualifier`; two type constraints is `TooManyTypes`. + +## 6. Pipelines, chains, commands + +```ebnf +pipeline-chain ::= assignment | pipeline { chain-op nl pipeline } [ '&' ] (* PipelineChainRule *) +chain-op ::= '&&' | '||' (* left nested PipelineChainAst *) +assignment ::= expression assignment-op nl statement (* RHS is a statement; assignment-op has the AssignmentOperator trait *) +pipeline ::= ( expression { redirection } | command ) { nl '|' nl command } (* PipelineRule; later elements must be commands *) +command ::= [ '&' | '.' ] command-name { command-element } (* CommandRule, mode C *) +command-name ::= Generic | Identifier | primary-expression (when preceded by '&' or '.') +command-element ::= command-parameter | command-argument | redirection | '--%' raw-text +command-parameter ::= Parameter [ ':' command-argument ] (* UsedColon => the argument follows *) +command-argument ::= argument-token { ',' nl argument-token } (* comma list => ArrayLiteralAst *) +argument-token ::= Generic | primary-expression | other-token-as-bareword (* GetCommandArgument *) +redirection ::= file-redirection command-argument-or-primary | merging-redirection (* RedirectionRule; '<' not supported *) +``` + +Notes: a pipeline continues across a newline only after `|` or when the next line starts with `|`. After the first element an expression is `ExpressionsMustBeFirstInPipeline`. Redirections are limited to `CommandBaseAst.MaxRedirections` slots; a stream may be redirected once (`StreamAlreadyRedirected`). + +## 7. Expressions + +```ebnf +expression ::= binary-expression [ '?' nl expression nl ':' nl expression ] (* ExpressionRule; ternary [≠G: branches are full expressions] *) +binary-expression ::= array-literal { binary-op nl array-literal } (* BinaryExpressionRule: precedence climbing, left assoc, levels in 02.2 §7.2 *) +array-literal ::= unary-expression { ',' nl unary-expression } (* ArrayLiteralRule; skipped when commas disabled *) +unary-expression ::= unary-op nl unary-expression | ',' nl unary-expression + | ( '++' | '--' ) nl unary-expression + | attribute-or-type-prefix unary-expression (* cast or attributed expression *) + | primary-expression [ '++' | '--' ] (* UnaryExpressionRule *) +unary-op ::= '-not' | '!' | '-bnot' | '+' | '-' | '-split' | '-join' +attribute-or-type-prefix ::= { '[' type-spec ']' | attribute } (* type literal; with '::' it is a static member target *) +primary-expression ::= value { member-access | element-access | invocation } (* PrimaryExpressionRule + CheckPostPrimaryExpressionOperators *) +value ::= Variable | SplattedVariable | Number | string-literal | expandable-string + | '(' nl pipeline-chain nl ')' (* ParenthesizedExpressionRule *) + | '$(' statement-list ')' | '@(' statement-list ')' (* SubExpressionRule, mode C *) + | '@{' hash-body '}' | '{' script-block '}' (* HashExpressionRule, ScriptBlockExpressionRule *) +member-access ::= ( '.' | '::' | '?.' ) nl member-name (* MemberAccessRule; no whitespace before operator *) +member-name ::= simple-name | string | unary-expression | primary-expression +invocation ::= member-access '(' [ expression { ',' expression } ] ')' | member-access script-block (* MemberInvokeRule, InvokeParamParenListRule; generic: '[' type { ',' type } ']' before '(' *) +element-access ::= ( '[' | '?[' ) nl expression nl ']' (* ElementAccessRule; commas allowed in index *) +hash-body ::= nl [ hash-entry { ( NewLine | ';' )+ hash-entry } ] nl +hash-entry ::= key-expression '=' nl statement (* GetKeyValuePair; key = simple-name | unary-expression *) +type-literal ::= '[' type-spec ']' +type-spec ::= type-name [ '[' { ',' } ']' | '[' type-spec { ',' type-spec } ']' ] [ ',' assembly-name ] (* TypeNameRule, mode T; array rank <= 32 *) +``` + +Notes (`FACT-SRC`): +- Binary operators are the kinds with the `BinaryOperator` trait except `&&`/`||`; operands are `array-literal`, so `1,2 -eq 3` is `(1,2) -eq 3`. +- A leading `[` in `unary-expression` is parsed as a list of attributes/type constraints; one trailing operand makes a cast (`ConvertExpressionAst`) or an attributed expression; a `::` after the type makes it a static member access target (`TypeExpressionAst`). +- `$using:x` in a variable token yields `UsingExpressionAst`. +- `argument` in an invocation is parsed with the comma operator disabled so commas separate arguments. +- Postfix `++`/`--` apply once. + +## 8. Contradictions between the in-source grammar comments and the code + +| Place | Comment says | Code does | +|---|---|---| +| `ExpressionRule` | ternary branches are `ternary-expression` | branches are full `ExpressionRule` results (right-nested) | +| `ThrowStatementRule` | `'throw' pipeline:opt` | uses `PipelineChainRule` (like `return`/`exit`) | +| `StatementRule` doc | "never returns null" | returns null at end of input | +| `PipelineChainRule` | text "if this feature is not enabled" | the chain operators are always enabled | +| `StatementRule` `statement-terminator` | each statement ends with a terminator | terminator optional, skipped by `StatementListRule` | + +Recorded in `CONTRADICTIONS.md` (CON-004, CON-005). + +## 9. Open questions / UNVERIFIED + +- Productions for `configuration`, dynamic keywords and DSC-specific parsing are summarized only (`EXTENDED`/DSC is out of scope for MicroteamShell). +- Behavior of `Label` before non-loop statements was read (`LabeledStatementRule`) but not exercised. +- All productions are `FACT-SRC` as syntax; their acceptance behavior on edge input is `UNVERIFIED` until run. + +Verification: each production re-derived from the method bodies listed (windows W7-W28 in the parser notes) and compared with the `// G` comments extracted by script from `Parser.cs`; differences are the table in §8; method names in comments exist in `data/parser-methods.json` (checked by `validate-refs` on the cited method ranges). diff --git a/docs-out/02-language/INSTRUCTION-SET.md b/docs-out/02-language/INSTRUCTION-SET.md new file mode 100644 index 0000000..54a7bb5 --- /dev/null +++ b/docs-out/02-language/INSTRUCTION-SET.md @@ -0,0 +1,340 @@ +--- +id: LANG-INSTR-001 +title: Instruction set +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/interpreter/Instruction.cs, src/System.Management.Automation/engine/interpreter/InstructionList.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/interpreter/_folder.md] +generated_from: [data/instructions.json, data/types.json] +--- + +# Instruction set + +> STATIC-ONLY — behavior not run-verified. The class list is `GENERATED` by `docs-out/tools/gen_T27.py` from `data/instructions.json` (`APPROX`: classes whose base chain reaches `Instruction`). Stack effects and semantics: [TIERING-POLICY.md](TIERING-POLICY.md) §6 and [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). Interpreter structure: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §7. + +## 1. Purpose and scope [SPEC] + +The interpreter (the "light compiler" in `engine/interpreter`) executes a flat array of `Instruction` objects over an object stack. This catalog lists the 183 instruction classes the scanner found (17 abstract bases) in 22 files. Generic families that are expanded by arity (`DynamicInstruction<…>`, `ActionCallInstruction<…>`, `FuncCallInstruction<…>`) are listed once per file by the scanner and described in section 4. + +## 2. Source map + +| path | role | +|---|---| +| `src/System.Management.Automation/engine/interpreter/Instruction.cs` | abstract `Instruction` base, `IInstructionProvider`, `NotInstruction` | +| `src/System.Management.Automation/engine/interpreter/InstructionList.cs` | `InstructionList` emitter and `InstructionArray` | +| `src/System.Management.Automation/engine/interpreter/Interpreter.cs` | `Interpreter.Run` loop | + +## 3. Instruction classes by file + +### `AddInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `AddInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L20-L151` | +| `AddInstruction.AddInt32` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L32-L42` | +| `AddInstruction.AddInt16` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L44-L54` | +| `AddInstruction.AddInt64` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L56-L66` | +| `AddInstruction.AddUInt16` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L68-L78` | +| `AddInstruction.AddUInt32` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L80-L90` | +| `AddInstruction.AddUInt64` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L92-L102` | +| `AddInstruction.AddSingle` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L104-L114` | +| `AddInstruction.AddDouble` | sealed | `AddInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L116-L126` | +| `AddOvfInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L153-L284` | +| `AddOvfInstruction.AddOvfInt32` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L165-L175` | +| `AddOvfInstruction.AddOvfInt16` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L177-L187` | +| `AddOvfInstruction.AddOvfInt64` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L189-L199` | +| `AddOvfInstruction.AddOvfUInt16` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L201-L211` | +| `AddOvfInstruction.AddOvfUInt32` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L213-L223` | +| `AddOvfInstruction.AddOvfUInt64` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L225-L235` | +| `AddOvfInstruction.AddOvfSingle` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L237-L247` | +| `AddOvfInstruction.AddOvfDouble` | sealed | `AddOvfInstruction` | `src/System.Management.Automation/engine/interpreter/AddInstruction.cs@L249-L259` | + +### `ArrayOperations.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `NewArrayInitInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ArrayOperations.cs@L18-L42` | +| `NewArrayInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ArrayOperations.cs@L44-L58` | +| `NewArrayBoundsInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ArrayOperations.cs@L60-L87` | +| `GetArrayItemInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ArrayOperations.cs@L89-L109` | +| `SetArrayItemInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ArrayOperations.cs@L111-L132` | + +### `CallInstruction.Generated.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `ActionCallInstruction` | sealed | `CallInstruction` | `src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs@L432-L456` | +| `FuncCallInstruction` | sealed | `CallInstruction` | `src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs@L692-L715` | +| `MethodInfoCallInstruction` | sealed | `CallInstruction` | `src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs@L942-L954` | + +### `CallInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `CallInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/CallInstruction.cs@L22-L264` | +| `MethodInfoCallInstruction` | sealed | `CallInstruction` | `src/System.Management.Automation/engine/interpreter/CallInstruction.cs@L266-L367` | + +### `ControlFlowInstructions.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `OffsetInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L29-L64` | +| `BranchFalseInstruction` | sealed | `OffsetInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L66-L92` | +| `BranchTrueInstruction` | sealed | `OffsetInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L94-L120` | +| `CoalescingBranchInstruction` | sealed | `OffsetInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L122-L150` | +| `BranchInstruction` | concrete | `OffsetInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L152-L196` | +| `IndexedBranchInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L198-L227` | +| `GotoInstruction` | sealed | `IndexedBranchInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L250-L303` | +| `EnterTryCatchFinallyInstruction` | sealed | `IndexedBranchInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L305-L447` | +| `EnterFinallyInstruction` | sealed | `IndexedBranchInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L452-L489` | +| `LeaveFinallyInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L494-L514` | +| `EnterExceptionHandlerInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L517-L547` | +| `LeaveExceptionHandlerInstruction` | sealed | `IndexedBranchInstruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L552-L590` | +| `LeaveFaultInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L595-L627` | +| `ThrowInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L629-L669` | +| `SwitchInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L671-L690` | +| `EnterLoopInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L692-L778` | +| `CompiledLoopInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs@L780-L794` | + +### `DivInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `DivInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L20-L151` | +| `DivInstruction.DivInt32` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L32-L42` | +| `DivInstruction.DivInt16` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L44-L54` | +| `DivInstruction.DivInt64` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L56-L66` | +| `DivInstruction.DivUInt16` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L68-L78` | +| `DivInstruction.DivUInt32` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L80-L90` | +| `DivInstruction.DivUInt64` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L92-L102` | +| `DivInstruction.DivSingle` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L104-L114` | +| `DivInstruction.DivDouble` | sealed | `DivInstruction` | `src/System.Management.Automation/engine/interpreter/DivInstruction.cs@L116-L126` | + +### `DynamicInstructionN.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `DynamicInstructionN` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/DynamicInstructionN.cs@L20-L77` | + +### `DynamicInstructions.Generated.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `DynamicInstruction` | concrete | `Instruction` | `src/System.Management.Automation/engine/interpreter/DynamicInstructions.Generated.cs@L103-L127` | + +### `DynamicSplatInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `DynamicSplatInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/DynamicSplatInstruction.cs@L23-L52` | + +### `EqualInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `EqualInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L20-L189` | +| `EqualInstruction.EqualBoolean` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L33-L40` | +| `EqualInstruction.EqualSByte` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L42-L49` | +| `EqualInstruction.EqualInt16` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L51-L58` | +| `EqualInstruction.EqualChar` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L60-L67` | +| `EqualInstruction.EqualInt32` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L69-L76` | +| `EqualInstruction.EqualInt64` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L78-L85` | +| `EqualInstruction.EqualByte` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L87-L94` | +| `EqualInstruction.EqualUInt16` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L96-L103` | +| `EqualInstruction.EqualUInt32` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L105-L112` | +| `EqualInstruction.EqualUInt64` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L114-L121` | +| `EqualInstruction.EqualSingle` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L123-L130` | +| `EqualInstruction.EqualDouble` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L132-L139` | +| `EqualInstruction.EqualReference` | sealed | `EqualInstruction` | `src/System.Management.Automation/engine/interpreter/EqualInstruction.cs@L141-L148` | + +### `FieldOperations.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `LoadStaticFieldInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/FieldOperations.cs@L21-L38` | +| `LoadFieldInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/FieldOperations.cs@L40-L59` | +| `StoreFieldInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/FieldOperations.cs@L61-L82` | +| `StoreStaticFieldInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/FieldOperations.cs@L84-L104` | + +### `GreaterThanInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `GreaterThanInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L20-L168` | +| `GreaterThanInstruction.GreaterThanSByte` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L32-L40` | +| `GreaterThanInstruction.GreaterThanInt16` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L42-L50` | +| `GreaterThanInstruction.GreaterThanChar` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L52-L60` | +| `GreaterThanInstruction.GreaterThanInt32` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L62-L70` | +| `GreaterThanInstruction.GreaterThanInt64` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L72-L80` | +| `GreaterThanInstruction.GreaterThanByte` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L82-L90` | +| `GreaterThanInstruction.GreaterThanUInt16` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L92-L100` | +| `GreaterThanInstruction.GreaterThanUInt32` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L102-L110` | +| `GreaterThanInstruction.GreaterThanUInt64` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L112-L120` | +| `GreaterThanInstruction.GreaterThanSingle` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L122-L130` | +| `GreaterThanInstruction.GreaterThanDouble` | sealed | `GreaterThanInstruction` | `src/System.Management.Automation/engine/interpreter/GreaterThanInstruction.cs@L132-L140` | + +### `Instruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `NotInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/Instruction.cs@L72-L87` | + +### `LessThanInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `LessThanInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L20-L168` | +| `LessThanInstruction.LessThanSByte` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L32-L40` | +| `LessThanInstruction.LessThanInt16` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L42-L50` | +| `LessThanInstruction.LessThanChar` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L52-L60` | +| `LessThanInstruction.LessThanInt32` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L62-L70` | +| `LessThanInstruction.LessThanInt64` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L72-L80` | +| `LessThanInstruction.LessThanByte` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L82-L90` | +| `LessThanInstruction.LessThanUInt16` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L92-L100` | +| `LessThanInstruction.LessThanUInt32` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L102-L110` | +| `LessThanInstruction.LessThanUInt64` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L112-L120` | +| `LessThanInstruction.LessThanSingle` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L122-L130` | +| `LessThanInstruction.LessThanDouble` | sealed | `LessThanInstruction` | `src/System.Management.Automation/engine/interpreter/LessThanInstruction.cs@L132-L140` | + +### `LocalAccess.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `LocalAccessInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L34-L49` | +| `LoadLocalInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L53-L73` | +| `LoadLocalBoxedInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L75-L90` | +| `LoadLocalFromClosureInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L92-L107` | +| `LoadLocalFromClosureBoxedInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L109-L124` | +| `AssignLocalInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L130-L151` | +| `StoreLocalInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L153-L173` | +| `AssignLocalBoxedInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L175-L192` | +| `StoreLocalBoxedInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L194-L211` | +| `AssignLocalToClosureInstruction` | sealed | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L213-L230` | +| `InitializeLocalInstruction` | abstract | `LocalAccessInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L237-L418` | +| `InitializeLocalInstruction.Reference` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L244-L266` | +| `InitializeLocalInstruction.ImmutableValue` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L268-L293` | +| `InitializeLocalInstruction.ImmutableBox` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L295-L316` | +| `InitializeLocalInstruction.ParameterBox` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L318-L330` | +| `InitializeLocalInstruction.Parameter` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L332-L359` | +| `InitializeLocalInstruction.MutableValue` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L361-L395` | +| `InitializeLocalInstruction.MutableBox` | sealed | `InitializeLocalInstruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L397-L417` | +| `RuntimeVariablesInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/LocalAccess.cs@L424-L453` | + +### `MulInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `MulInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L20-L151` | +| `MulInstruction.MulInt32` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L32-L42` | +| `MulInstruction.MulInt16` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L44-L54` | +| `MulInstruction.MulInt64` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L56-L66` | +| `MulInstruction.MulUInt16` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L68-L78` | +| `MulInstruction.MulUInt32` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L80-L90` | +| `MulInstruction.MulUInt64` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L92-L102` | +| `MulInstruction.MulSingle` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L104-L114` | +| `MulInstruction.MulDouble` | sealed | `MulInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L116-L126` | +| `MulOvfInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L153-L284` | +| `MulOvfInstruction.MulOvfInt32` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L165-L175` | +| `MulOvfInstruction.MulOvfInt16` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L177-L187` | +| `MulOvfInstruction.MulOvfInt64` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L189-L199` | +| `MulOvfInstruction.MulOvfUInt16` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L201-L211` | +| `MulOvfInstruction.MulOvfUInt32` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L213-L223` | +| `MulOvfInstruction.MulOvfUInt64` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L225-L235` | +| `MulOvfInstruction.MulOvfSingle` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L237-L247` | +| `MulOvfInstruction.MulOvfDouble` | sealed | `MulOvfInstruction` | `src/System.Management.Automation/engine/interpreter/MulInstruction.cs@L249-L259` | + +### `NotEqualInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `NotEqualInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L20-L189` | +| `NotEqualInstruction.NotEqualBoolean` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L33-L40` | +| `NotEqualInstruction.NotEqualSByte` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L42-L49` | +| `NotEqualInstruction.NotEqualInt16` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L51-L58` | +| `NotEqualInstruction.NotEqualChar` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L60-L67` | +| `NotEqualInstruction.NotEqualInt32` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L69-L76` | +| `NotEqualInstruction.NotEqualInt64` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L78-L85` | +| `NotEqualInstruction.NotEqualByte` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L87-L94` | +| `NotEqualInstruction.NotEqualUInt16` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L96-L103` | +| `NotEqualInstruction.NotEqualUInt32` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L105-L112` | +| `NotEqualInstruction.NotEqualUInt64` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L114-L121` | +| `NotEqualInstruction.NotEqualSingle` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L123-L130` | +| `NotEqualInstruction.NotEqualDouble` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L132-L139` | +| `NotEqualInstruction.NotEqualReference` | sealed | `NotEqualInstruction` | `src/System.Management.Automation/engine/interpreter/NotEqualInstruction.cs@L141-L148` | + +### `NumericConvertInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `NumericConvertInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/NumericConvertInstruction.cs@L18-L284` | +| `NumericConvertInstruction.Unchecked` | sealed | `NumericConvertInstruction` | `src/System.Management.Automation/engine/interpreter/NumericConvertInstruction.cs@L38-L159` | +| `NumericConvertInstruction.Checked` | sealed | `NumericConvertInstruction` | `src/System.Management.Automation/engine/interpreter/NumericConvertInstruction.cs@L162-L283` | + +### `PowerShellInstructions.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `UpdatePositionInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/PowerShellInstructions.cs@L18-L50` | + +### `StackOperations.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `LoadObjectInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/StackOperations.cs@L21-L42` | +| `LoadCachedObjectInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/StackOperations.cs@L44-L70` | +| `PopInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/StackOperations.cs@L72-L90` | +| `DupInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/StackOperations.cs@L92-L112` | + +### `SubInstruction.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `SubInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L20-L151` | +| `SubInstruction.SubInt32` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L32-L42` | +| `SubInstruction.SubInt16` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L44-L54` | +| `SubInstruction.SubInt64` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L56-L66` | +| `SubInstruction.SubUInt16` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L68-L78` | +| `SubInstruction.SubUInt32` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L80-L90` | +| `SubInstruction.SubUInt64` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L92-L102` | +| `SubInstruction.SubSingle` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L104-L114` | +| `SubInstruction.SubDouble` | sealed | `SubInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L116-L126` | +| `SubOvfInstruction` | abstract | `Instruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L153-L284` | +| `SubOvfInstruction.SubOvfInt32` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L165-L175` | +| `SubOvfInstruction.SubOvfInt16` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L177-L187` | +| `SubOvfInstruction.SubOvfInt64` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L189-L199` | +| `SubOvfInstruction.SubOvfUInt16` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L201-L211` | +| `SubOvfInstruction.SubOvfUInt32` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L213-L223` | +| `SubOvfInstruction.SubOvfUInt64` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L225-L235` | +| `SubOvfInstruction.SubOvfSingle` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L237-L247` | +| `SubOvfInstruction.SubOvfDouble` | sealed | `SubOvfInstruction` | `src/System.Management.Automation/engine/interpreter/SubInstruction.cs@L249-L259` | + +### `TypeOperations.cs` + +| Class | Kind | Base | Source | +|---|---|---|---| +| `CreateDelegateInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/TypeOperations.cs@L21-L55` | +| `NewInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/TypeOperations.cs@L57-L99` | +| `DefaultValueInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/TypeOperations.cs@L101-L119` | +| `TypeIsInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/TypeOperations.cs@L121-L140` | +| `TypeAsInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/TypeOperations.cs@L142-L170` | +| `TypeEqualsInstruction` | sealed | `Instruction` | `src/System.Management.Automation/engine/interpreter/TypeOperations.cs@L172-L196` | + +## 4. Generated families (not enumerated by the scanner) + +| Family | Members | Evidence | +|---|---|---| +| `DynamicInstruction<…>` | 16 generic classes, 0..15 call arguments (`DynamicInstruction` … ``) | `src/System.Management.Automation/engine/interpreter/DynamicInstructions.Generated.cs` (notes W1-W2) | +| `ActionCallInstruction<…>` | 10 classes, 0..9 arguments | `src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs` | +| `FuncCallInstruction<…>` | 10 classes, 0..9 arguments | same file | +| `GetArrayItemInstruction`, `SetArrayItemInstruction`, `TypeIsInstruction`, `TypeAsInstruction`, `DefaultValueInstruction`, `NewArrayInstruction`, `NewArrayInitInstruction` | created per element type on demand by `InstructionFactory` (15 pre-seeded types) | `src/System.Management.Automation/engine/interpreter/InstructionFactory.cs` | + +Counts for these families come from the generated-file notes (`FACT-SRC`), not from `data/instructions.json`. + +Verification: instruction class list reconciles with `data/instructions.json` (count above); abstract/sealed taken from the class modifiers; family sizes from the three generated source files read in full (notes `.work/notes/src_System.Management.Automation_engine_interpreter_*`). + diff --git a/docs-out/02-language/IR-SPEC.md b/docs-out/02-language/IR-SPEC.md new file mode 100644 index 0000000..20b43d9 --- /dev/null +++ b/docs-out/02-language/IR-SPEC.md @@ -0,0 +1,193 @@ +--- +id: LANG-IR-001 +title: Intermediate representation +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/interpreter/LightCompiler.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [] +--- + +# Intermediate representation + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. The IR is the shape of the expression trees `Compiler` produces; it is the contract between [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md), the interpreter ([INSTRUCTION-SET.md](INSTRUCTION-SET.md)) and the JIT ([TIERING-POLICY.md](TIERING-POLICY.md)). + +## 1. Purpose and scope [SPEC] + +PowerShell has no bytecode of its own. Its IR is a `System.Linq.Expressions` tree: one `Expression>` per named block, containing ordinary LINQ nodes, DLR `DynamicExpression` call sites for PowerShell operations, and three PowerShell-specific extension nodes. This document specifies the lambda contract, the runtime objects the tree reads and writes, the protocols the tree implements (output, `$?`, error handling, positions), and which LINQ node kinds it uses. Anything labelled "protocol" is behavior a new implementation must reproduce even if it does not use expression trees ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md)). + +## 2. Source map + +| path | type | role | +|---|---|---| +| src/System.Management.Automation/engine/parser/Compiler.cs::FunctionContext@L784-L814 | class | the one parameter of every lambda | +| src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661 | class | builds the trees | +| src/System.Management.Automation/engine/parser/Compiler.cs::PowerShellLoopExpression@L6947-L6989 | class | extension node: a loop that the interpreter can promote to JIT | +| src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7023 | class | extension node: loop iteration counter | +| src/System.Management.Automation/engine/parser/Compiler.cs::UpdatePositionExpr@L7025-L7081 | class | extension node: sequence point and breakpoint check | +| src/System.Management.Automation/engine/parser/Compiler.cs::ExpressionCache@L659-L718 | class | shared constant nodes | + +## 3. Responsibilities and non-responsibilities + +Describes the tree shape, not the AST → tree rules (see [COMPILATION-MAP.md](COMPILATION-MAP.md)) and not binder behavior (T26, planned BINDER-CATALOG). + +## 4. Key types and contracts [IMPL] + +### 4.1 The lambda + +Signature `Action`; name = function name plus a clause suffix (``, ``, ``, ``, ``, ``), `` for anonymous blocks. Single-expression lambdas have the signature `Func`. Built by `CompileSingleLambda` (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleLambda@L2552-L2626): + +1. `context` = `funcContext._executionContext`, `locals` = `(TupleType)funcContext._localsTuple` (declared as the first block variables; the interpreter reads them at fixed frame slots 0 and 1, see src/System.Management.Automation/engine/interpreter/InterpretedFrame.cs::InterpretedFrame@L28-L332). +2. Prolog (`GenerateFunctionProlog`): store the function name into the context, run an entry position update, `context._debugger.EnterScriptFunction(funcContext)`; for class members also redirect the output pipe to a null pipe and bind `$this`; constructors initialize property defaults. +3. Optional (root block only): load `using` directives and define classes (`GenerateTypesAndUsings`). +4. Body: the statement list (`CompileStatementListWithTraps`). +5. A `returnTarget` label, then the epilog (exit position update). +6. Whole body wrapped in `try … finally { context._debugger.ExitScriptFunction() }`; if the block has traps (or contains a `trap` statement), `try … catch (ReturnException) {}` is added around it. + +### 4.2 `FunctionContext` (what the IR can read and write) + +Fields: `_scriptBlock`, `_file`, `_debuggerHidden`, `_debuggerStepThrough`, `_sequencePoints` (extents), `_executionContext`, `_outputPipe`, `_breakPoints`, `_boundBreakpoints`, `_currentSequencePointIndex`, `_localsTuple`, `_traps` (stack of trap handler tuples), `_functionName`; methods `PushTrapHandlers`, `PopTrapHandlers` (src/System.Management.Automation/engine/parser/Compiler.cs::FunctionContext@L784-L814). + +### 4.3 Locals + +A `MutableTuple` whose type comes from variable analysis ([02.4-semantic-analysis.md](02.4-semantic-analysis.md) §7.2). Slot access is a chain of property reads over the tuple access path. Variables with `TupleIndex < 0` are accessed by name through `VariableOps.GetVariableValue/SetVariableValue(path, context, ast)`. + +## 5. Data model — the protocols + +- **Output protocol.** A statement that produces a value writes it to `funcContext._outputPipe` (`CallAddPipe`: non-enumerable static types are added directly, others through `PSPipeWriterBinder`). The expression-tree value of a pure expression is *not* the output. Assignments produce no output. `++`/`--` produce no output. To collect the output of a sub-statement (conditions, assignments, `@()`, `$()`), the tree swaps `_outputPipe` for a new `Pipe` over a `List`, compiles the statement, and restores the old pipe in a `finally` (src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureAstResults@L2264-L2355). +- **`$?` protocol.** After a statement that "succeeds trivially" (`ShouldSetExecutionStatusToSuccess`: pure expressions and assignments of pure expressions, `$()` with no statements, …) the tree sets `context.QuestionMarkVariableValue = true`; commands set it at run time. +- **Error-action protocol.** Every statement is wrapped so that a non-flow-control exception is passed to `ExceptionHandlingOps.CheckActionPreference(funcContext, e)`; if it returns, execution resumes at the next statement; flow-control exceptions (`FlowControlException`) are rethrown untouched. Detail: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6.4. +- **Position protocol.** Before each statement/condition the tree stores the sequence-point index into `funcContext._currentSequencePointIndex` and, if the debugger is active (`_debuggingMode > 0`), calls `OnSequencePointHit`. `UpdatePositionExpr` reduces to `DebugInfo` + that store for the JIT, and to `UpdatePositionInstruction` for the interpreter. +- **Flow-control protocol.** `break`/`continue` to a loop in the same lambda are `Goto` labels; otherwise `BreakException(label)`/`ContinueException(label)` are thrown and caught by the enclosing loop's catch blocks; `return` is a `Return` to the lambda's label (or `ReturnException` inside trap lambdas); `exit` throws the exception from `PipelineOps.GetExitException`; `throw` converts via `ExceptionHandlingOps.ConvertToException`. +- **Dynamic operations.** Arithmetic other than double-double, comparisons, conversions, member/index/invoke operations, enumeration, and pipeline writes are `DynamicExpression` sites using the PowerShell binders (`PSBinaryOperationBinder`, `PSConvertBinder`, `PSGetMemberBinder`, …); each emitted site has its own `CallSite` (per-site caches). Binder semantics: BINDER-CATALOG (planned T26). + +## 6. Control flow + +Not applicable at this level: the tree is straight code with labels and try blocks; loop and statement shapes are shown in [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §6. + +## 7. Algorithms + +### 7.1 LINQ node kinds used + +Usage counts of `Expression.(` calls in `Compiler.cs` (`GENERATED`, `APPROX`: calls, not emitted nodes) with the light compiler's support (hand-read from `LightCompiler.cs`; `FACT-SRC`): + + +| LINQ factory | Calls in Compiler.cs | Interpreter (LightCompiler) support | +|---|---:|---| +| `Expression.Constant` | 130 | yes | +| `Expression.Call` | 102 | yes (by-ref or value-type instance call forces full JIT) | +| `Expression.Block` | 89 | yes (defines locals) | +| `Expression.Assign` | 72 | yes (local, parameter, index, member; not compound) | +| `Expression.Dynamic` | 62 | yes (DynamicInstruction<...> per site) | +| `Expression.Label` | 29 | yes | +| `Expression.Variable` | 21 | yes | +| `Expression.NewArrayInit` | 15 | yes | +| `Expression.Catch` | 14 | yes (no filters) | +| `Expression.IfThen` | 14 | yes | +| `Expression.Field` | 13 | yes (literal/static/instance via reflection) | +| `Expression.New` | 12 | yes (by-ref ctor forces JIT) | +| `Expression.Property` | 12 | yes (getter call) | +| `Expression.TryFinally` | 12 | yes | +| `Expression.Goto` | 9 | yes | +| `Expression.Not` | 9 | yes (bool only) | +| `Expression.Convert` | 8 | numeric convert, otherwise no-op | +| `Expression.Parameter` | 8 | yes | +| `Expression.Throw` | 8 | yes | +| `Expression.Rethrow` | 7 | yes | +| `Expression.SwitchCase` | 6 | int constants only | +| `Expression.TryCatch` | 6 | yes | +| `Expression.Condition` | 5 | yes | +| `Expression.NotEqual` | 5 | numeric/reference | +| `Expression.Equal` | 4 | numeric/reference | +| `Expression.TryCatchFinally` | 4 | yes | +| `Expression.IfThenElse` | 3 | yes | +| `Expression.Switch` | 3 | int test values, no comparison method | +| `Expression.AndAlso` | 2 | bool only | +| `Expression.Empty` | 2 | yes | +| `Expression.TypeIs` | 2 | yes | +| `Expression.Add` | 1 | yes | +| `Expression.ArrayAccess` | 1 | yes (indexer) | +| `Expression.Break` | 1 | yes (goto) | +| `Expression.Continue` | 1 | yes (goto) | +| `Expression.DebugInfo` | 1 | recorded only | +| `Expression.Divide` | 1 | yes | +| `Expression.GreaterThan` | 1 | numeric | +| `Expression.Lambda` | 1 | nested lambdas via closure | +| `Expression.Multiply` | 1 | yes | +| `Expression.OrElse` | 1 | bool only | +| `Expression.Return` | 1 | yes (goto) | +| `Expression.Subtract` | 1 | yes | +| `Expression.SymbolDocument` | 1 | n/a (debug info document) | + + +Extension nodes (`ExpressionType.Extension`, implement `IInstructionProvider` so the interpreter can compile them without reducing): `PowerShellLoopExpression` (a loop; `AddInstructions` emits its body and finishes the loop counter), `EnterLoopExpression` (emits `EnterLoopInstruction` only if the loop statement count is below the limit), `UpdatePositionExpr` (emits `UpdatePositionInstruction`). + +### 7.2 What the interpreter cannot handle + +`LightCompiler` throws for: binary `And`/`Or`/`Modulo`/`Power`/shifts/`ExclusiveOr` without a method, unary operators other than `Not` on `bool`/`Convert`/`TypeAs`, `Loop` (body commented out), `Switch` with non-int or non-constant tests, `Try` with filters or fault blocks, `ListInit`, `MemberInit`, `Quote`, compound assignments, `Coalesce` with a conversion, and `Invocation` of a lambda operand; it **forces a full JIT compile** for by-ref calls/constructors and instance calls on non-primitive value types ([TIERING-POLICY.md](TIERING-POLICY.md) §7.2). The IR avoids the throwing shapes by construction (`INFERENCE`; PowerShell uses dynamic sites for operators other than double arithmetic and its own loop node instead of `Loop`). + +## 8. Observable behavior [SPEC] + +The IR is not observable by scripts. Observable consequences are the protocols in §5. No golden sample: STATIC-ONLY. + +## 9. Error handling and edge cases + +Single-expression lambdas swallow `FlowControlException` and rethrow `TerminateException` (`GetExpressionValue`). A statement list with ≥ 2 statements uses an integer dispatch with `goto`; after a handled error execution resumes at the *following* statement. The JIT is known to be expensive for many `try/catch` regions (comment in `CompileStatementListWithTraps`). + +## 10. Concurrency and threading + +A tree is built once per (script block, variant); the produced delegate is shared by all invocations and all clones. Per-call state lives in `FunctionContext` and the locals tuple (new per call). + +## 11. Extension points and customization hooks + +New node type → add a `VisitX` to `Compiler` and an `IInstructionProvider` if a custom node is needed. New operator → either a dynamic site with an existing binder or a `ParserOps`/`StringOps` helper call. + +## 12. Dependencies + +`System.Linq.Expressions`, `System.Dynamic` (call sites), the PowerShell binders and runtime helper classes listed in `CachedReflectionInfo` (src/System.Management.Automation/engine/parser/Compiler.cs::CachedReflectionInfo@L28-L657, about 250 cached `MethodInfo`/`FieldInfo` entries grouped by helper class). + +## 13. Performance notes + +Constant caches (`ExpressionCache.Constant(int)` caches -1..100), reflection info cache, native `double` arithmetic only when both static types are `double`, per-site binder caches. + +## 14. Security considerations (mandatory) + +The IR embeds the AST nodes and extents as constants (error positions, `$using` keys). The `data` statement sets `context.LanguageMode` to restricted around its body and restores it in a `finally`. Constrained-language enforcement is not in the IR but in the binders and runtime helpers (T26, SEC documents). + +## 15. Enterprise considerations + +Debuggability: `DebugInfo` + sequence points + `EnterScriptFunction/ExitScriptFunction` give step/breakpoint support in both tiers. Missing: no IR dump facility in release builds (`DebugViewWriter.cs` exists for debug view of trees, `UNVERIFIED` use). + +## 16. Tests + +Not read (VB-014). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| IR = LINQ expression trees | `FACT-SRC` | reuse of the DLR call sites and the .NET JIT; no custom backend | +| output via side effect on a pipe object, not via tree values | `FACT-SRC` | statements compose like commands; needs explicit capture for conditions and assignments | +| per-statement exception wrapper with resume | `FACT-SRC` | non-terminating-error semantics inside script code; cost in tree size and JIT time | +| custom extension nodes for loops and positions | `FACT-SRC` | the interpreter can promote a running loop | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-IR-01** [ADAPT] (area: NLD-RUN) A tree-walker over the AST can replace the IR, provided it implements the protocols in §5 (output pipe, `$?`, error-action resume, positions, flow control). Why: the observable behavior is defined by the protocols, not by expression trees. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleLambda@L2552-L2626. +- **LSN-IR-02** [REPLICATE] (area: NLD-PIP) Statements write to an output pipe; assignments, `++`/`--` and conditions do not; capturing output swaps the current pipe and restores it in a `finally`. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureAstResults@L2264-L2355, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitCommandExpression@L4268-L4279. +- **LSN-IR-03** [REPLICATE] (area: NLD-ERR) Wrap each statement so that non-flow-control exceptions go to the error-action policy and execution continues with the next statement when the policy says so. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268. +- **LSN-IR-04** [ADAPT] (area: NLD-RUN) Represent break/continue/return as control-flow signals inside the evaluator, and as exceptions only when crossing function or script-block boundaries. Why: PowerShell uses labels in-lambda and exceptions across lambdas; a tree-walker has no lambdas but still crosses script-block calls. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateBreakOrContinue@L5511-L5551. +- **LSN-IR-05** [REPLICATE] (area: NLD-RUN) Keep a per-statement sequence-point index in the call frame so errors and the debugger can report positions. Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::UpdatePositionExpr@L7025-L7081. + +## 19. Open questions / UNVERIFIED + +- Use of `DebugViewWriter.cs`; whether it is reachable in release builds. +- Whether any `Expression.Switch` shape used by the compiler (catch dispatch) is accepted by the light compiler's int-only switch (3 uses of `Expression.Switch`; read: catch handler and statement dispatch use int tests). + +Verification: node-kind counts generated by `tools/gen_T25.py` from `Compiler.cs`; support column cross-read from `LightCompiler.cs` notes W3-W7; prolog/epilog/wrapper re-read from `CompileSingleLambda` (L2552-2626) and `GenerateFunctionProlog` (L2903-2959). diff --git a/docs-out/02-language/TIERING-POLICY.md b/docs-out/02-language/TIERING-POLICY.md new file mode 100644 index 0000000..4f6f909 --- /dev/null +++ b/docs-out/02-language/TIERING-POLICY.md @@ -0,0 +1,158 @@ +--- +id: LANG-TIERING-001 +title: Tiering policy +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-CMP-001, CF-CMP-002, CF-CMP-003] +sources: [src/System.Management.Automation/engine/parser/Compiler.cs, src/System.Management.Automation/engine/interpreter/LightCompiler.cs, src/System.Management.Automation/engine/interpreter/LightLambda.cs, src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs, src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs, src/System.Management.Automation/engine/interpreter/LoopCompiler.cs, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/interpreter/_folder.md] +generated_from: [] +--- + +# Tiering policy + +> STATIC-ONLY — behavior not run-verified. `FACT-SRC` unless marked. This document is the single owner of the compile-versus-interpret rule and of every tiering threshold. Related: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md), [IR-SPEC.md](IR-SPEC.md), [INSTRUCTION-SET.md](INSTRUCTION-SET.md), [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). + +## 1. Purpose and scope [SPEC] + +PowerShell compiles each script block twice over: first into a **LINQ expression tree** (the IR), then into something executable. The executable form is chosen by *tiering*: new code is **interpreted** by a small stack machine; hot code is **JIT-compiled** by `System.Linq.Expressions` (`LambdaExpression.Compile()`). A second, independent choice is **optimized vs unoptimized** code (fixed-slot locals vs name lookup). This document states the rules and the numbers. Terminology: *compile* = AST → IR; *JIT-compile* = IR → delegate; *interpret* = run IR through the instruction interpreter; *tier* = interpret or JIT ([GLOSSARY](../GLOSSARY.md)). + +## 2. Source map + +| path | type | role | +|---|---|---| +| src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964 | method | per script block: variable analysis, build lambdas, choose the tiering mode | +| src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2097 | method | build the tiered delegate for one lambda | +| src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2051 | class | IR → instructions; holds the call-count threshold | +| src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs::LightDelegateCreator@L30-L219 | class | owns the interpreter and the (lazy) compiled delegate | +| src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda@L43-L302 | class | per-delegate call counter; triggers JIT | +| src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterLoopInstruction@L692-L778 | class | per-loop iteration counter; triggers loop JIT | +| src/System.Management.Automation/engine/interpreter/LoopCompiler.cs::LoopCompiler@L29-L384 | class | JIT-compiles one loop out of an interpreted frame | +| src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData@L41-L540 | class | optimized/unoptimized compile and caches | + +## 3. Responsibilities and non-responsibilities + +Chooses *when* code is interpreted or JIT-compiled and *which variant* runs. Does not change observable results: both tiers must agree ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md)). + +## 4. Key types and contracts [IMPL] + +`CompileInterpretChoice { NeverCompile, AlwaysCompile, CompileOnDemand }` (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompileInterpretChoice@L25-L30). `LightCompiler(int compilationThreshold)`: a negative threshold means `DefaultCompilationThreshold`; zero or one means synchronous compilation (src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2051, src/System.Management.Automation/engine/interpreter/Interpreter.cs::Interpreter@L31-L109). + +## 5. Data model + +- One delegate per named block (`DynamicParam`, `Begin`, `Process`, `End`, `Clean`) **per variant** (optimized, unoptimized): ten fields on `CompiledScriptBlockData`. +- `LightLambda` holds a mutable `int` call counter (copied from the interpreter's threshold when the delegate is created); `EnterLoopInstruction` holds a mutable loop-iteration counter. +- `LightDelegateCreator` holds the lazily built compiled delegate shared by all `LightLambda` instances created from it. + +## 6. Control flow + +```mermaid +flowchart TD + A[ScriptBlock.InvokeWithPipeImpl] --> B{debugging off and new scope?} + B -->|yes| O[optimized variant] + B -->|no| U[unoptimized variant] + O --> C{any AllScope variable in slot map?} + C -->|yes| U + C -->|no| D[delegate for clause] + U --> D + D --> E{compiled delegate ready?} + E -->|yes| J[run JIT code] + E -->|no| F[run interpreter] + F --> G{call counter reaches 0?} + G -->|yes| H[queue JIT on thread pool] + F --> L{EnterLoop counter reaches 0?} + L -->|yes| M[queue loop JIT; replace instruction] +``` + +## 7. Algorithm — the rule + +### 7.1 Which variant (CF-CMP-003) + +**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used for script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables. + +Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.InvokeWithPipeImpl@L935-L959 (`runOptimized = context._debuggingMode <= 0 && createLocalScope`), src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552 (constructor), src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847, src/System.Management.Automation/engine/parser/VariableAnalysis.cs::FindAllVariablesVisitor@L44-L323. + +### 7.2 Interpret versus JIT (CF-CMP-001) + +**Compile-versus-interpret rule.** Every named block is turned into one expression-tree lambda. At creation of the delegate (`Compiler.CompileTree`), the tiering mode is `NeverCompile` if the script block's total statement count (`_stmtCount`) is greater than 300, otherwise `CompileOnDemand`; `AlwaysCompile` exists but no caller passes it. The delegate starts as an **interpreted** `LightLambda` with a call counter of 32 (`CompileOnDemand`, threshold -1 mapped to `DefaultCompilationThreshold`) or `int.MaxValue` (`NeverCompile`). Each call decrements the counter; the call that reads 0 (the 33rd) queues JIT compilation of the whole lambda on a thread-pool thread and is itself still interpreted; once the compiled delegate exists, every later call uses it. A lambda is **never interpreted** (JIT-compiled at once, synchronously) when the light compiler meets a method call with a by-ref parameter, an instance call on a non-primitive value type, or a constructor with a by-ref parameter; and trap-handler lambdas and single-expression lambdas are always built with `LambdaExpression.Compile()` (never interpreted). Loops are promoted separately (CF-CMP-002). + +Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964 (L2050 mode selection), src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2097, src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2051 (threshold, `_forceCompile`), src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246, src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs::LightDelegateCreator.CreateDelegate@L78-L81, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTrap@L2529-L2550, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleExpression@L2204-L2226. + +### 7.3 Loop promotion + +A loop may be JIT-compiled **while it is running** in the interpreter. `GenerateWhileLoop`/`GenerateDoLoop` emit an `EnterLoopExpression` at the loop top; it becomes an `EnterLoopInstruction` only if the loop's own statement count is below the limit. On each arrival at the loop top the counter is decremented; when the read value is 0 the loop body is compiled by `LoopCompiler` (queued on the thread pool unless the interpreter is in synchronous mode) and the instruction in the instruction array is replaced by a `CompiledLoopInstruction` that runs the rest of the loop in JIT code and returns the instruction offset to continue at. The frame's local array, closure boxes and the frame object itself are handed over; the evaluation stack is not (a loop is a statement). `NeverCompile` does not disable loop promotion. + +### 7.4 Constants and limits (CF-CMP-002) + +**Tiering thresholds and limits.** Statement count above which a script block is never JIT-compiled by call count: 300 (`_stmtCount > 300`, `Compiler.Compile`). Interpreted calls before JIT: 32 (`LightCompiler.DefaultCompilationThreshold`), so JIT starts on the 33rd call; `int.MaxValue` for `NeverCompile`; threshold 0 or 1 means synchronous compilation (never produced by `Compiler`). Loop iterations before loop JIT: 16 (constructor argument of `EnterLoopInstruction` in `EnterLoopExpression.AddInstructions`), so the loop JIT starts on the 17th arrival at the loop top. Loop statement count at or above which a loop is never promoted: 300 (`LoopStatementCount < 300` required to emit the loop counter). Maximum delegate parameters of an interpreted lambda: 16 (`LightLambda.MaxParameters`). Compile helper cache of delegate `Run` factories: 100 entries (`CacheDict` in `LightLambda`). Parsed-script cache (shares compiled delegates between clones): cleared completely when it holds more than 1024 entries (`ScriptBlock.CacheScriptBlock`). + +Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L961-L964, src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2051, src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7023, src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterLoopInstruction@L692-L778, src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda@L43-L302, src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.CacheScriptBlock@L609-L637. + +### 7.5 Concurrency of the decision + +Counters are decremented without locks on the hot path (racing threads may lose decrements or both enter the "compile" branch); `LightDelegateCreator.Compile` takes a lock and runs once. JIT runs on thread-pool threads, so *which tier executes a given call is not deterministic*; results must not depend on it. `FACT-SRC` (comments in `LightLambda.TryGetCompiled`). + +## 8. Observable behavior [SPEC] + +Only indirect: performance, stack traces (interpreted frames are grouped by `InterpretedFrame`), debugger behavior (debugging forces unoptimized code). No golden sample: STATIC-ONLY. Whether results are identical across tiers is the subject of [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). + +## 9. Error handling and edge cases + +The light compiler throws `NotImplementedException`/`NotSupportedException` for IR shapes it does not support (see [IR-SPEC.md](IR-SPEC.md) §5); no handler for that was found in `Compiler.CompileTree` (`UNVERIFIED`: the IR emitted by `Compiler` is believed to avoid them, VB-013). A compile exception on the thread-pool thread is not observed by the caller (`INFERENCE`). + +## 10. Concurrency and threading + +See §7.5. The compiled delegate is published by `LightDelegateCreator` under a lock; `LightLambda` and `EnterLoopInstruction` counters are plain fields. + +## 11. Extension points and customization hooks + +None public. `InternalTestHooks` can disable the script block cache. `CompileSynchronously` (interpreter threshold ≤ 1) exists for tests. + +## 12. Dependencies + +Inbound: `ScriptBlock` invocation, `PSScriptCmdlet`, `Compiler`. Outbound: `System.Linq.Expressions`, `ThreadPool`. + +## 13. Performance notes + +Tiering exists to keep start-up cheap (scripts that run once never pay JIT cost) and still speed up hot functions and loops. The 300-statement guards avoid very expensive JIT of huge blocks (comments in `EnterLoopExpression.AddInstructions` and `Compiler.CompileStatementListWithTraps`: JIT cost grows badly with the number of `try/catch` blocks). + +## 14. Security considerations (mandatory) + +Tiering has no security role, but two items matter: (a) debugging switches to unoptimized code, so a debugger sees variables by name; (b) compiled delegates are shared between `ScriptBlock` clones, so any state captured in them must not be session-specific (the script block cache returns clones for that reason). + +## 15. Enterprise considerations + +Observability: ETW `CompileStart/Stop` around `ReallyCompile` (src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.ReallyCompile@L163-L195). Missing: no switch to force one tier, no metrics on promotions, no way to bound background JIT threads. + +## 16. Tests + +No tiering-specific test was read. `test/powershell/Language/` loops and functions exercise both tiers implicitly (`UNVERIFIED`, VB-014). + +## 17. Design decisions and trade-offs + +| Decision | Label | Consequence | +|---|---|---| +| interpret first, JIT after 32 calls | `FACT-SRC` | cheap start; hot code speeds up; nondeterministic tier per call | +| loops promoted independently after 16 iterations | `FACT-SRC` | long-running top-level loops get fast without waiting for a second call | +| statement-count cutoffs (300) | `FACT-SRC` | bounds JIT cost; large scripts stay interpreted | +| optimized variant only without debugging and with a new scope | `FACT-SRC` | dot-sourcing and debugging pay name lookups | + +## 18. Guidance for MicroteamShell [LESSON] + +- **LSN-TIER-01** [REPLICATE] (area: NLD-RUN) Start with a tree-walking evaluator only; it is a valid first tier (all observable behavior is defined by the IR semantics, not by the JIT). `INFERENCE`: PowerShell's tiers are equivalent by design ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md)). Evidence: src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2097. +- **LSN-TIER-02** [ADAPT] (area: NLD-RUN) If a second tier is ever added, trigger it by call count with a deterministic (synchronous) policy first, and keep results independent of the tier. Why: PowerShell's asynchronous promotion makes tier choice nondeterministic. Evidence: src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246. +- **LSN-TIER-03** [REPLICATE] (area: NLD-SCP) Keep two variable-storage strategies visible in the design: fast slots when the scope is private and analyzable, dynamic lookup when debugging, dot-sourcing, or `AllScope`/`New-Variable`-style access is possible. Evidence: src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83. +- **LSN-TIER-04** [AVOID] (area: NLD-RUN) Do not copy background (thread-pool) compilation of hot loops with an in-place instruction replacement; it needs the interpreter and compiled code to share frames exactly. Replace by an explicit, synchronous optimization step if a loop optimizer is wanted. Evidence: src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterLoopInstruction@L692-L778. DEV: DEV-TIER-060 + +## 19. Open questions / UNVERIFIED + +- Where `NotImplementedException` from the light compiler would surface (VB-013). +- Whether `AlwaysCompile` is reachable through any other assembly (searched `engine/`; only `Compiler.CompileTree` references it). +- Effect of tiering on `$?`/stack traces is not tested here. + +Verification: thresholds re-read from `Compiler.cs` (L2050-2097, L7008-7020), `LightCompiler.cs` (L271-301), `LightLambda.cs` (L197-246), `CompiledScriptBlock.cs` (L57-83, L1004-1010) and `ControlFlowInstructions.cs` (`EnterLoopInstruction`, notes W1-W3); `rg "AlwaysCompile|NeverCompile|CompileOnDemand" engine/` shows the three enum members and the three uses in `Compiler.cs` only. diff --git a/docs-out/02-language/TOKEN-CATALOG.md b/docs-out/02-language/TOKEN-CATALOG.md new file mode 100644 index 0000000..839a639 --- /dev/null +++ b/docs-out/02-language/TOKEN-CATALOG.md @@ -0,0 +1,434 @@ +--- +id: LANG-TOKCAT-001 +title: Token catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/token.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/CharTraits.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/enums.json, data/tables.json] +--- + +# Token catalog + +> STATIC-ONLY — behavior not run-verified. Every table below is `GENERATED` by `docs-out/tools/gen_T21.py` from `data/enums.json` and `data/tables.json` (`APPROX`: the arrays are parsed from verbatim source text; lengths are asserted equal to the `TokenKind` value range). Narrative lives in [02.1-tokenizer.md](02.1-tokenizer.md). Label for every row: `FACT-SRC` (read in source) + `APPROX` (extraction). + +## 1. Purpose and scope [SPEC] + +Complete list of token kinds, token flags, keywords, dash operators, character classes, number-suffix flags and tokenizer modes of the PowerShell tokenizer (v7.6.9). Counts: `TokenKind` has 157 named members spread over the value range 0..170; `TokenFlags` has 28 members. + +## 2. Source map + +| path | type | role | +|---|---|---| +| `src/System.Management.Automation/engine/parser/token.cs::TokenKind@L18-L595` | enum | every token kind | +| `src/System.Management.Automation/engine/parser/token.cs::TokenFlags@L601-L752` | `[Flags]` enum | token traits | +| `src/System.Management.Automation/engine/parser/token.cs::TokenTraits.s_staticTokenFlags@L759-L957` | table | static flags per kind | +| `src/System.Management.Automation/engine/parser/token.cs::TokenTraits.s_tokenText@L959-L1211` | table | default text per kind | +| `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.s_keywordText@L625-L639` | table | keyword spelling | +| `src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer._operatorText@L657-L674` | table | dash-operator spelling | +| `src/System.Management.Automation/engine/parser/CharTraits.cs::CharTraits@L28-L89` | `[Flags]` enum | per-character class bits | + + +## 3. TokenKind (every member) [SPEC] + +Value gaps are reserved slots (placeholders `Reserved slot N` in the static tables); they are listed so the value range is complete. `Text` is the entry of `s_tokenText`; `Static flags` is the entry of `s_staticTokenFlags`. + +| Value | Name | Source line | Text | Static flags | +|---:|---|---:|---|---| +| 0 | `Unknown` | L27 | `unknown` | None | +| 1 | `Variable` | L33 | `var` | None | +| 2 | `SplattedVariable` | L40 | `@var` | None | +| 3 | `Parameter` | L46 | `param` | None | +| 4 | `Number` | L52 | `number` | None | +| 5 | `Label` | L58 | `label` | None | +| 6 | `Identifier` | L63 | `ident` | None | +| 7 | `Generic` | L72 | `generic` | None | +| 8 | `NewLine` | L75 | `newline` | ParseModeInvariant | +| 9 | `LineContinuation` | L78 | `line continuation` | ParseModeInvariant | +| 10 | `Comment` | L81 | `comment` | ParseModeInvariant | +| 11 | `EndOfInput` | L84 | `eof` | ParseModeInvariant | +| 12 | `StringLiteral` | L94 | `sqstr` | ParseModeInvariant | +| 13 | `StringExpandable` | L101 | `dqstr` | ParseModeInvariant | +| 14 | `HereStringLiteral` | L107 | `sq here string` | ParseModeInvariant | +| 15 | `HereStringExpandable` | L114 | `dq here string` | ParseModeInvariant | +| 16 | `LParen` | L122 | `(` | ParseModeInvariant | +| 17 | `RParen` | L126 | `)` | ParseModeInvariant | +| 18 | `LCurly` | L129 | `{` | ParseModeInvariant | +| 19 | `RCurly` | L132 | `}` | ParseModeInvariant | +| 20 | `LBracket` | L135 | `[` | None | +| 21 | `RBracket` | L138 | `]` | ParseModeInvariant | +| 22 | `AtParen` | L142 | `@(` | ParseModeInvariant | +| 23 | `AtCurly` | L145 | `@{` | ParseModeInvariant | +| 24 | `DollarParen` | L149 | `$(` | ParseModeInvariant | +| 25 | `Semi` | L152 | `;` | ParseModeInvariant | +| 26 | `AndAnd` | L159 | `&&` | ParseModeInvariant | +| 27 | `OrOr` | L162 | `\|\|` | ParseModeInvariant | +| 28 | `Ampersand` | L165 | `&` | SpecialOperator \| ParseModeInvariant | +| 29 | `Pipe` | L168 | `\|` | SpecialOperator \| ParseModeInvariant | +| 30 | `Comma` | L171 | `,` | UnaryOperator \| ParseModeInvariant | +| 31 | `MinusMinus` | L174 | `--` | UnaryOperator \| PrefixOrPostfixOperator \| DisallowedInRestrictedMode | +| 32 | `PlusPlus` | L177 | `++` | UnaryOperator \| PrefixOrPostfixOperator \| DisallowedInRestrictedMode | +| 33 | `DotDot` | L180 | `..` | BinaryOperator \| BinaryPrecedenceRange \| DisallowedInRestrictedMode | +| 34 | `ColonColon` | L183 | `::` | SpecialOperator \| DisallowedInRestrictedMode | +| 35 | `Dot` | L186 | `.` | SpecialOperator \| DisallowedInRestrictedMode | +| 36 | `Exclaim` | L189 | `!` | UnaryOperator \| CanConstantFold | +| 37 | `Multiply` | L192 | `*` | BinaryOperator \| BinaryPrecedenceMultiply \| CanConstantFold | +| 38 | `Divide` | L195 | `/` | BinaryOperator \| BinaryPrecedenceMultiply \| CanConstantFold | +| 39 | `Rem` | L198 | `%` | BinaryOperator \| BinaryPrecedenceMultiply \| CanConstantFold | +| 40 | `Plus` | L201 | `+` | BinaryOperator \| BinaryPrecedenceAdd \| UnaryOperator \| CanConstantFold | +| 41 | `Minus` | L204 | `-` | BinaryOperator \| BinaryPrecedenceAdd \| UnaryOperator \| CanConstantFold | +| 42 | `Equals` | L207 | `=` | AssignmentOperator | +| 43 | `PlusEquals` | L210 | `+=` | AssignmentOperator | +| 44 | `MinusEquals` | L213 | `-=` | AssignmentOperator | +| 45 | `MultiplyEquals` | L216 | `*=` | AssignmentOperator | +| 46 | `DivideEquals` | L219 | `/=` | AssignmentOperator | +| 47 | `RemainderEquals` | L222 | `%=` | AssignmentOperator | +| 48 | `Redirection` | L225 | `redirection` | DisallowedInRestrictedMode | +| 49 | `RedirectInStd` | L228 | `<` | ParseModeInvariant \| DisallowedInRestrictedMode | +| 50 | `Format` | L231 | `-f` | BinaryOperator \| BinaryPrecedenceFormat \| DisallowedInRestrictedMode | +| 51 | `Not` | L234 | `-not` | UnaryOperator \| CanConstantFold | +| 52 | `Bnot` | L238 | `-bnot` | UnaryOperator \| CanConstantFold | +| 53 | `And` | L241 | `-and` | BinaryOperator \| BinaryPrecedenceLogical \| CanConstantFold | +| 54 | `Or` | L244 | `-or` | BinaryOperator \| BinaryPrecedenceLogical \| CanConstantFold | +| 55 | `Xor` | L247 | `-xor` | BinaryOperator \| BinaryPrecedenceLogical \| CanConstantFold | +| 56 | `Band` | L250 | `-band` | BinaryOperator \| BinaryPrecedenceBitwise \| CanConstantFold | +| 57 | `Bor` | L254 | `-bor` | BinaryOperator \| BinaryPrecedenceBitwise \| CanConstantFold | +| 58 | `Bxor` | L258 | `-bxor` | BinaryOperator \| BinaryPrecedenceBitwise \| CanConstantFold | +| 59 | `Join` | L261 | `-join` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 60 | `Ieq` | L265 | `-eq` | BinaryOperator \| BinaryPrecedenceComparison | +| 61 | `Ine` | L269 | `-ne` | BinaryOperator \| BinaryPrecedenceComparison | +| 62 | `Ige` | L273 | `-ge` | BinaryOperator \| BinaryPrecedenceComparison | +| 63 | `Igt` | L277 | `-gt` | BinaryOperator \| BinaryPrecedenceComparison | +| 64 | `Ilt` | L281 | `-lt` | BinaryOperator \| BinaryPrecedenceComparison | +| 65 | `Ile` | L285 | `-le` | BinaryOperator \| BinaryPrecedenceComparison | +| 66 | `Ilike` | L289 | `-ilike` | BinaryOperator \| BinaryPrecedenceComparison | +| 67 | `Inotlike` | L293 | `-inotlike` | BinaryOperator \| BinaryPrecedenceComparison | +| 68 | `Imatch` | L297 | `-imatch` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 69 | `Inotmatch` | L301 | `-inotmatch` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 70 | `Ireplace` | L305 | `-ireplace` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 71 | `Icontains` | L309 | `-icontains` | BinaryOperator \| BinaryPrecedenceComparison | +| 72 | `Inotcontains` | L313 | `-inotcontains` | BinaryOperator \| BinaryPrecedenceComparison | +| 73 | `Iin` | L317 | `-iin` | BinaryOperator \| BinaryPrecedenceComparison | +| 74 | `Inotin` | L321 | `-inotin` | BinaryOperator \| BinaryPrecedenceComparison | +| 75 | `Isplit` | L325 | `-isplit` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 76 | `Ceq` | L329 | `-ceq` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 77 | `Cne` | L333 | `-cne` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 78 | `Cge` | L337 | `-cge` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 79 | `Cgt` | L341 | `-cgt` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 80 | `Clt` | L345 | `-clt` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 81 | `Cle` | L348 | `-cle` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 82 | `Clike` | L352 | `-clike` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 83 | `Cnotlike` | L356 | `-cnotlike` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 84 | `Cmatch` | L360 | `-cmatch` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 85 | `Cnotmatch` | L364 | `-cnotmatch` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 86 | `Creplace` | L368 | `-creplace` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 87 | `Ccontains` | L372 | `-ccontains` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 88 | `Cnotcontains` | L376 | `-cnotcontains` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 89 | `Cin` | L380 | `-cin` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 90 | `Cnotin` | L384 | `-cnotin` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 91 | `Csplit` | L388 | `-csplit` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 92 | `Is` | L391 | `-is` | BinaryOperator \| BinaryPrecedenceComparison | +| 93 | `IsNot` | L394 | `-isnot` | BinaryOperator \| BinaryPrecedenceComparison | +| 94 | `As` | L397 | `-as` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 95 | `PostfixPlusPlus` | L400 | `++` | UnaryOperator \| PrefixOrPostfixOperator \| DisallowedInRestrictedMode | +| 96 | `PostfixMinusMinus` | L403 | `--` | UnaryOperator \| PrefixOrPostfixOperator \| DisallowedInRestrictedMode | +| 97 | `Shl` | L407 | `-shl` | BinaryOperator \| BinaryPrecedenceComparison \| CanConstantFold | +| 98 | `Shr` | L411 | `-shr` | BinaryOperator \| BinaryPrecedenceComparison \| CanConstantFold | +| 99 | `Colon` | L414 | `:` | SpecialOperator \| DisallowedInRestrictedMode | +| 100 | `QuestionMark` | L417 | `?` | TernaryOperator \| DisallowedInRestrictedMode | +| 101 | `QuestionQuestionEquals` | L420 | `??=` | AssignmentOperator | +| 102 | `QuestionQuestion` | L423 | `??` | BinaryOperator \| BinaryPrecedenceCoalesce | +| 103 | `QuestionDot` | L426 | `?.` | SpecialOperator \| DisallowedInRestrictedMode | +| 104 | `QuestionLBracket` | L429 | `?[` | None | +| 105 | *(unnamed: reserved slot)* | — | `` | None | +| 106 | *(unnamed: reserved slot)* | — | `` | None | +| 107 | *(unnamed: reserved slot)* | — | `` | None | +| 108 | *(unnamed: reserved slot)* | — | `` | None | +| 109 | *(unnamed: reserved slot)* | — | `` | None | +| 110 | *(unnamed: reserved slot)* | — | `` | None | +| 111 | *(unnamed: reserved slot)* | — | `` | None | +| 112 | *(unnamed: reserved slot)* | — | `` | None | +| 113 | *(unnamed: reserved slot)* | — | `` | None | +| 114 | *(unnamed: reserved slot)* | — | `` | None | +| 115 | *(unnamed: reserved slot)* | — | `` | None | +| 116 | *(unnamed: reserved slot)* | — | `` | None | +| 117 | *(unnamed: reserved slot)* | — | `` | None | +| 118 | *(unnamed: reserved slot)* | — | `` | None | +| 119 | `Begin` | L436 | `begin` | Keyword \| ScriptBlockBlockName | +| 120 | `Break` | L439 | `break` | Keyword \| StatementDoesntSupportAttributes | +| 121 | `Catch` | L442 | `catch` | Keyword | +| 122 | `Class` | L445 | `class` | Keyword | +| 123 | `Continue` | L448 | `continue` | Keyword \| StatementDoesntSupportAttributes | +| 124 | `Data` | L451 | `data` | Keyword \| StatementDoesntSupportAttributes | +| 125 | `Define` | L454 | `define` | Keyword \| StatementDoesntSupportAttributes | +| 126 | `Do` | L457 | `do` | Keyword \| StatementDoesntSupportAttributes | +| 127 | `Dynamicparam` | L461 | `dynamicparam` | Keyword \| ScriptBlockBlockName | +| 128 | `Else` | L464 | `else` | Keyword | +| 129 | `ElseIf` | L467 | `elseif` | Keyword | +| 130 | `End` | L470 | `end` | Keyword \| ScriptBlockBlockName | +| 131 | `Exit` | L473 | `exit` | Keyword \| StatementDoesntSupportAttributes | +| 132 | `Filter` | L476 | `filter` | Keyword \| StatementDoesntSupportAttributes | +| 133 | `Finally` | L479 | `finally` | Keyword | +| 134 | `For` | L482 | `for` | Keyword \| StatementDoesntSupportAttributes | +| 135 | `Foreach` | L486 | `foreach` | Keyword \| StatementDoesntSupportAttributes | +| 136 | `From` | L489 | `from` | Keyword \| StatementDoesntSupportAttributes | +| 137 | `Function` | L492 | `function` | Keyword \| StatementDoesntSupportAttributes | +| 138 | `If` | L495 | `if` | Keyword \| StatementDoesntSupportAttributes | +| 139 | `In` | L498 | `in` | Keyword | +| 140 | `Param` | L502 | `param` | Keyword | +| 141 | `Process` | L505 | `process` | Keyword \| ScriptBlockBlockName | +| 142 | `Return` | L508 | `return` | Keyword \| StatementDoesntSupportAttributes | +| 143 | `Switch` | L511 | `switch` | Keyword \| StatementDoesntSupportAttributes | +| 144 | `Throw` | L514 | `throw` | Keyword \| StatementDoesntSupportAttributes | +| 145 | `Trap` | L517 | `trap` | Keyword \| StatementDoesntSupportAttributes | +| 146 | `Try` | L520 | `try` | Keyword \| StatementDoesntSupportAttributes | +| 147 | `Until` | L523 | `until` | Keyword | +| 148 | `Using` | L526 | `using` | Keyword \| StatementDoesntSupportAttributes | +| 149 | `Var` | L529 | `var` | Keyword \| StatementDoesntSupportAttributes | +| 150 | `While` | L532 | `while` | Keyword \| StatementDoesntSupportAttributes | +| 151 | `Workflow` | L535 | `workflow` | Keyword \| StatementDoesntSupportAttributes | +| 152 | `Parallel` | L538 | `parallel` | Keyword \| StatementDoesntSupportAttributes | +| 153 | `Sequence` | L541 | `sequence` | Keyword \| StatementDoesntSupportAttributes | +| 154 | `InlineScript` | L544 | `inlinescript` | Keyword \| StatementDoesntSupportAttributes | +| 155 | `Configuration` | L547 | `configuration` | Keyword | +| 156 | `DynamicKeyword` | L550 | `` | Keyword | +| 157 | `Public` | L553 | `public` | Keyword | +| 158 | `Private` | L556 | `private` | Keyword | +| 159 | `Static` | L559 | `static` | Keyword | +| 160 | `Interface` | L562 | `interface` | Keyword | +| 161 | `Enum` | L565 | `enum` | Keyword | +| 162 | `Namespace` | L568 | `namespace` | Keyword | +| 163 | `Module` | L571 | `module` | Keyword | +| 164 | `Type` | L574 | `type` | Keyword | +| 165 | `Assembly` | L577 | `assembly` | Keyword | +| 166 | `Command` | L580 | `command` | Keyword | +| 167 | `Hidden` | L583 | `hidden` | Keyword | +| 168 | `Base` | L586 | `base` | Keyword | +| 169 | `Default` | L589 | `default` | Keyword | +| 170 | `Clean` | L592 | `clean` | Keyword \| ScriptBlockBlockName | + +## 4. TokenFlags (every member) [SPEC] + +| Name | Value | Source line | +|---|---|---:| +| `None` | `0x00000000` | L606 | +| `BinaryPrecedenceLogical` | `0x1` | L613 | +| `BinaryPrecedenceBitwise` | `0x2` | L618 | +| `BinaryPrecedenceComparison` | `0x5` | L625 | +| `BinaryPrecedenceCoalesce` | `0x7` | L630 | +| `BinaryPrecedenceAdd` | `0x9` | L635 | +| `BinaryPrecedenceMultiply` | `0xa` | L640 | +| `BinaryPrecedenceFormat` | `0xc` | L645 | +| `BinaryPrecedenceRange` | `0xd` | L650 | +| `BinaryPrecedenceMask` | `0x0000000f` | L657 | +| `Keyword` | `0x00000010` | L662 | +| `ScriptBlockBlockName` | `0x00000020` | L667 | +| `BinaryOperator` | `0x00000100` | L672 | +| `UnaryOperator` | `0x00000200` | L677 | +| `CaseSensitiveOperator` | `0x00000400` | L682 | +| `TernaryOperator` | `0x00000800` | L687 | +| `SpecialOperator` | `0x00001000` | L692 | +| `AssignmentOperator` | `0x00002000` | L697 | +| `ParseModeInvariant` | `0x00008000` | L702 | +| `TokenInError` | `0x00010000` | L707 | +| `DisallowedInRestrictedMode` | `0x00020000` | L712 | +| `PrefixOrPostfixOperator` | `0x00040000` | L717 | +| `CommandName` | `0x00080000` | L722 | +| `MemberName` | `0x00100000` | L727 | +| `TypeName` | `0x00200000` | L732 | +| `AttributeName` | `0x00400000` | L737 | +| `CanConstantFold` | `0x00800000` | L746 | +| `StatementDoesntSupportAttributes` | `0x01000000` | L751 | + +## 5. Keywords (`Tokenizer.s_keywordText` / `s_keywordTokenKind`) [SPEC] + +Parallel arrays; recognition is case-insensitive (`s_keywordTable` is built with `OrdinalIgnoreCase`, see [02.1-tokenizer.md](02.1-tokenizer.md) §6). `inlinescript` is only a keyword inside a workflow context. + +| # | Spelling | TokenKind | Static flags of the kind | +|---:|---|---|---| +| 1 | `elseif` | `ElseIf` | Keyword | +| 2 | `if` | `If` | Keyword \| StatementDoesntSupportAttributes | +| 3 | `else` | `Else` | Keyword | +| 4 | `switch` | `Switch` | Keyword \| StatementDoesntSupportAttributes | +| 5 | `foreach` | `Foreach` | Keyword \| StatementDoesntSupportAttributes | +| 6 | `from` | `From` | Keyword \| StatementDoesntSupportAttributes | +| 7 | `in` | `In` | Keyword | +| 8 | `for` | `For` | Keyword \| StatementDoesntSupportAttributes | +| 9 | `while` | `While` | Keyword \| StatementDoesntSupportAttributes | +| 10 | `until` | `Until` | Keyword | +| 11 | `do` | `Do` | Keyword \| StatementDoesntSupportAttributes | +| 12 | `try` | `Try` | Keyword \| StatementDoesntSupportAttributes | +| 13 | `catch` | `Catch` | Keyword | +| 14 | `finally` | `Finally` | Keyword | +| 15 | `trap` | `Trap` | Keyword \| StatementDoesntSupportAttributes | +| 16 | `data` | `Data` | Keyword \| StatementDoesntSupportAttributes | +| 17 | `return` | `Return` | Keyword \| StatementDoesntSupportAttributes | +| 18 | `continue` | `Continue` | Keyword \| StatementDoesntSupportAttributes | +| 19 | `break` | `Break` | Keyword \| StatementDoesntSupportAttributes | +| 20 | `exit` | `Exit` | Keyword \| StatementDoesntSupportAttributes | +| 21 | `throw` | `Throw` | Keyword \| StatementDoesntSupportAttributes | +| 22 | `begin` | `Begin` | Keyword \| ScriptBlockBlockName | +| 23 | `process` | `Process` | Keyword \| ScriptBlockBlockName | +| 24 | `end` | `End` | Keyword \| ScriptBlockBlockName | +| 25 | `dynamicparam` | `Dynamicparam` | Keyword \| ScriptBlockBlockName | +| 26 | `function` | `Function` | Keyword \| StatementDoesntSupportAttributes | +| 27 | `filter` | `Filter` | Keyword \| StatementDoesntSupportAttributes | +| 28 | `param` | `Param` | Keyword | +| 29 | `class` | `Class` | Keyword | +| 30 | `define` | `Define` | Keyword \| StatementDoesntSupportAttributes | +| 31 | `var` | `Var` | Keyword \| StatementDoesntSupportAttributes | +| 32 | `using` | `Using` | Keyword \| StatementDoesntSupportAttributes | +| 33 | `workflow` | `Workflow` | Keyword \| StatementDoesntSupportAttributes | +| 34 | `parallel` | `Parallel` | Keyword \| StatementDoesntSupportAttributes | +| 35 | `sequence` | `Sequence` | Keyword \| StatementDoesntSupportAttributes | +| 36 | `inlinescript` | `InlineScript` | Keyword \| StatementDoesntSupportAttributes | +| 37 | `configuration` | `Configuration` | Keyword | +| 38 | `public` | `Public` | Keyword | +| 39 | `private` | `Private` | Keyword | +| 40 | `static` | `Static` | Keyword | +| 41 | `interface` | `Interface` | Keyword | +| 42 | `enum` | `Enum` | Keyword | +| 43 | `namespace` | `Namespace` | Keyword | +| 44 | `module` | `Module` | Keyword | +| 45 | `type` | `Type` | Keyword | +| 46 | `assembly` | `Assembly` | Keyword | +| 47 | `command` | `Command` | Keyword | +| 48 | `hidden` | `Hidden` | Keyword | +| 49 | `base` | `Base` | Keyword | +| 50 | `default` | `Default` | Keyword | +| 51 | `clean` | `Clean` | Keyword \| ScriptBlockBlockName | + +Keyword rows: **51** (`GENERATED`). `TokenKind.DynamicKeyword` has no row; dynamic (DSC) keywords come from `DynamicKeyword.DynamicKeywords`. + +## 6. Dash operators (`Tokenizer._operatorText` / `s_operatorTokenKind`) [SPEC] + +Spelling is stored **without** the leading dash. Recognition happens only in expression mode (see [02.1-tokenizer.md](02.1-tokenizer.md) §6.4). + +| # | Spelling | TokenKind | Static flags of the kind | +|---:|---|---|---| +| 1 | `-bnot` | `Bnot` | UnaryOperator \| CanConstantFold | +| 2 | `-not` | `Not` | UnaryOperator \| CanConstantFold | +| 3 | `-eq` | `Ieq` | BinaryOperator \| BinaryPrecedenceComparison | +| 4 | `-ieq` | `Ieq` | BinaryOperator \| BinaryPrecedenceComparison | +| 5 | `-ceq` | `Ceq` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 6 | `-ne` | `Ine` | BinaryOperator \| BinaryPrecedenceComparison | +| 7 | `-ine` | `Ine` | BinaryOperator \| BinaryPrecedenceComparison | +| 8 | `-cne` | `Cne` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 9 | `-ge` | `Ige` | BinaryOperator \| BinaryPrecedenceComparison | +| 10 | `-ige` | `Ige` | BinaryOperator \| BinaryPrecedenceComparison | +| 11 | `-cge` | `Cge` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 12 | `-gt` | `Igt` | BinaryOperator \| BinaryPrecedenceComparison | +| 13 | `-igt` | `Igt` | BinaryOperator \| BinaryPrecedenceComparison | +| 14 | `-cgt` | `Cgt` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 15 | `-lt` | `Ilt` | BinaryOperator \| BinaryPrecedenceComparison | +| 16 | `-ilt` | `Ilt` | BinaryOperator \| BinaryPrecedenceComparison | +| 17 | `-clt` | `Clt` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 18 | `-le` | `Ile` | BinaryOperator \| BinaryPrecedenceComparison | +| 19 | `-ile` | `Ile` | BinaryOperator \| BinaryPrecedenceComparison | +| 20 | `-cle` | `Cle` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 21 | `-like` | `Ilike` | BinaryOperator \| BinaryPrecedenceComparison | +| 22 | `-ilike` | `Ilike` | BinaryOperator \| BinaryPrecedenceComparison | +| 23 | `-clike` | `Clike` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 24 | `-notlike` | `Inotlike` | BinaryOperator \| BinaryPrecedenceComparison | +| 25 | `-inotlike` | `Inotlike` | BinaryOperator \| BinaryPrecedenceComparison | +| 26 | `-cnotlike` | `Cnotlike` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 27 | `-match` | `Imatch` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 28 | `-imatch` | `Imatch` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 29 | `-cmatch` | `Cmatch` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 30 | `-notmatch` | `Inotmatch` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 31 | `-inotmatch` | `Inotmatch` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 32 | `-cnotmatch` | `Cnotmatch` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 33 | `-replace` | `Ireplace` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 34 | `-ireplace` | `Ireplace` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 35 | `-creplace` | `Creplace` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 36 | `-contains` | `Icontains` | BinaryOperator \| BinaryPrecedenceComparison | +| 37 | `-icontains` | `Icontains` | BinaryOperator \| BinaryPrecedenceComparison | +| 38 | `-ccontains` | `Ccontains` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 39 | `-notcontains` | `Inotcontains` | BinaryOperator \| BinaryPrecedenceComparison | +| 40 | `-inotcontains` | `Inotcontains` | BinaryOperator \| BinaryPrecedenceComparison | +| 41 | `-cnotcontains` | `Cnotcontains` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 42 | `-in` | `Iin` | BinaryOperator \| BinaryPrecedenceComparison | +| 43 | `-iin` | `Iin` | BinaryOperator \| BinaryPrecedenceComparison | +| 44 | `-cin` | `Cin` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 45 | `-notin` | `Inotin` | BinaryOperator \| BinaryPrecedenceComparison | +| 46 | `-inotin` | `Inotin` | BinaryOperator \| BinaryPrecedenceComparison | +| 47 | `-cnotin` | `Cnotin` | BinaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator | +| 48 | `-split` | `Isplit` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 49 | `-isplit` | `Isplit` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 50 | `-csplit` | `Csplit` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| CaseSensitiveOperator \| DisallowedInRestrictedMode | +| 51 | `-isnot` | `IsNot` | BinaryOperator \| BinaryPrecedenceComparison | +| 52 | `-is` | `Is` | BinaryOperator \| BinaryPrecedenceComparison | +| 53 | `-as` | `As` | BinaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 54 | `-f` | `Format` | BinaryOperator \| BinaryPrecedenceFormat \| DisallowedInRestrictedMode | +| 55 | `-and` | `And` | BinaryOperator \| BinaryPrecedenceLogical \| CanConstantFold | +| 56 | `-band` | `Band` | BinaryOperator \| BinaryPrecedenceBitwise \| CanConstantFold | +| 57 | `-or` | `Or` | BinaryOperator \| BinaryPrecedenceLogical \| CanConstantFold | +| 58 | `-bor` | `Bor` | BinaryOperator \| BinaryPrecedenceBitwise \| CanConstantFold | +| 59 | `-xor` | `Xor` | BinaryOperator \| BinaryPrecedenceLogical \| CanConstantFold | +| 60 | `-bxor` | `Bxor` | BinaryOperator \| BinaryPrecedenceBitwise \| CanConstantFold | +| 61 | `-join` | `Join` | BinaryOperator \| UnaryOperator \| BinaryPrecedenceComparison \| DisallowedInRestrictedMode | +| 62 | `-shl` | `Shl` | BinaryOperator \| BinaryPrecedenceComparison \| CanConstantFold | +| 63 | `-shr` | `Shr` | BinaryOperator \| BinaryPrecedenceComparison \| CanConstantFold | + +Dash-operator rows: **63** (`GENERATED`). + +## 7. CharTraits (character-class bits) [SPEC] + +| Name | Value | Source line | +|---|---|---:| +| `None` | `0x0000` | L33 | +| `IdentifierStart` | `0x0002` | L38 | +| `MultiplierStart` | `0x0004` | L43 | +| `TypeSuffix` | `0x0008` | L48 | +| `Whitespace` | `0x0010` | L53 | +| `Newline` | `0x0020` | L58 | +| `HexDigit` | `0x0040` | L63 | +| `Digit` | `0x0080` | L68 | +| `VarNameFirst` | `0x0100` | L73 | +| `ForceStartNewToken` | `0x0200` | L78 | +| `ForceStartNewAssemblyNameSpecToken` | `0x0400` | L83 | +| `ForceStartNewTokenAfterNumber` | `0x0800` | L88 | + +## 8. NumberSuffixFlags and NumberFormat [SPEC] + +| Name | Value | Source line | +|---|---|---:| +| `None` | `0x0` | L506 | +| `Unsigned` | `0x1` | L511 | +| `SignedByte` | `0x2` | L516 | +| `UnsignedByte` | `0x3` | L522 | +| `Short` | `0x4` | L527 | +| `UnsignedShort` | `0x5` | L533 | +| `Long` | `0x8` | L538 | +| `UnsignedLong` | `0x9` | L544 | +| `Decimal` | `0x10` | L549 | +| `BigInteger` | `0x20` | L554 | + +| NumberFormat | Value | Source line | +|---|---|---:| +| `Decimal` | `0x0` | L565 | +| `Hex` | `0x1` | L570 | +| `Binary` | `0x2` | L575 | + +## 9. TokenizerMode [SPEC] + +| Name | Source line | +|---|---:| +| `Command` | L491 | +| `Expression` | L492 | +| `TypeName` | L493 | +| `Signature` | L494 | + +## 10. Cross-document links + +Narrative, mode-transition table and lexical rules: [02.1-tokenizer.md](02.1-tokenizer.md). Parser consumers of each kind: [02.2-parser.md](02.2-parser.md) (planned, T22). Token subclasses: [02.1-tokenizer.md](02.1-tokenizer.md) §4. + + +Verification: row counts asserted equal to the `TokenKind` value range (171) and to `data/numbers.json` keys `TOKENKIND_MEMBERS`, `TOKENFLAGS_MEMBERS`; keyword and operator arrays asserted pairwise equal length (`gen_T21.py`). Static-flag and text arrays were also read in the source windows recorded in `.work/notes/` (token.cs W3-W4). + diff --git a/docs-out/09-synthesis/deviation-register.md b/docs-out/09-synthesis/deviation-register.md index f44af78..37be615 100644 --- a/docs-out/09-synthesis/deviation-register.md +++ b/docs-out/09-synthesis/deviation-register.md @@ -20,6 +20,13 @@ All rows are `PROPOSED` (the only status this run may write). `APPROVED`/`REJECT | DEV-ID | status | baseline behavior (CF/GS) | proposed behavior | benefit | compatibility cost | affected CFs | golden samples that would change | verdict locations | |---|---|---|---|---|---|---|---|---| +| DEV-AST-020 | PROPOSED | AST nodes carry mutable compiler caches (TupleIndex, RuntimeUsingIndex, TypeDefinitionAst.Type) and implement compile-time assignment protocols (ISupportsAssignment/IAssignableValue). | AST stays purely syntactic; slot indexes and assignment code generation live in compiler side tables keyed by node. | Immutable shareable trees; clean separation of parser and compiler; easier tooling reuse. | None observable by scripts; internal refactor only. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-AST-07 (02-language/02.3-ast.md) | | DEV-CMD-120 | PROPOSED | Windows runtimes ship `Get-Counter`, `Get-WinEvent`, `New-WinEvent` in the default command set (assembly Microsoft.PowerShell.Commands.Diagnostics). | Do not ship OS-specific diagnostics cmdlets in the default set; offer them as optional modules. | Smaller default attack surface and a portable base command set. | Scripts calling these cmdlets need the optional module installed. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-DIAG-01 (10-command-modules/Microsoft.PowerShell.Commands.Diagnostics.md) | | DEV-CMD-121 | PROPOSED | WSMan client cmdlets and provider (assembly Microsoft.WSMan.Management) are part of the Windows command set. | Omit WSMan commands; remoting transport becomes pluggable behind one session abstraction. | No dependency on a Windows-only protocol stack in the base product. | WSMan-based automation breaks; an optional transport module could restore it. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-WSMAN-01 (10-command-modules/Microsoft.WSMan.Management.md) | | DEV-CMD-122 | PROPOSED | CIM/WMI cmdlets (assembly Microsoft.Management.Infrastructure.CimCmdlets) ship on Windows. | Provide CIM/WMI access only as an optional adapter module. | Keeps the base command set free of Windows management stacks. | Windows management scripts need the optional module. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-CIM-01 (10-command-modules/Microsoft.Management.Infrastructure.CimCmdlets.md) | +| DEV-ENG-040 | PROPOSED | Compilation of a top-level script block runs an AMSI scan, admin-block check and suspicious-content hashing. | Compilation is policy-free; an optional policy hook runs before execution. | Smaller engine core; platform-independent. | No AMSI integration or forced logging on suspicious content. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-ENG-07 (02-language/02.5-compiler-and-engine.md) | +| DEV-EQV-061 | PROPOSED | Interpreter keeps typed arithmetic/comparison instruction families inherited from the DLR. | Do not port them; numeric semantics come from the evaluator's conversion rules. | Removes dead and defective code. | None observable (instructions unreachable from PowerShell IR). | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-EQV-03 (02-language/EQUIVALENCE-SPEC.md) | +| DEV-GRM-010 | PROPOSED | The parser resolves `using assembly` paths against the filesystem and may create a runspace for `configuration` while parsing. | Parsing is pure: directives are recorded in the AST and resolved in a later phase. | Deterministic, side-effect-free parse; safe for tooling and untrusted text. | Errors for a missing assembly appear at a later phase than in PowerShell. | CF-GRM-001 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-PARSER-07 (02-language/02.2-parser.md) | +| DEV-LEX-001 | PROPOSED | Tokenizer detects signature-block lookalikes and processes #requires comment lines at parse time. | Omit both from the tokenizer; if wanted, handle pragmas in a dedicated post-tokenize pass. | Keeps the tokenizer a pure text-to-token function; removes security-sensitive fuzzy matching from the hot path. | Scripts using #requires get no version/module check; signed-script trailer detection unavailable. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TOK-07 (02-language/02.1-tokenizer.md) | +| DEV-SEM-030 | PROPOSED | `using module` is resolved at parse time by running Get-Module -ListAvailable in a PowerShell instance. | The directive is recorded; module import happens in a later phase. | Parsing has no side effects or I/O. | Missing-module errors appear later than in PowerShell. | - | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-SEM-06 (02-language/02.4-semantic-analysis.md) | +| DEV-TIER-060 | PROPOSED | Hot loops are promoted from interpreter to JIT in the background mid-execution via in-place instruction replacement. | No mid-execution promotion; any optimizer runs synchronously at defined points. | Deterministic execution, simpler frames. | Long top-level loops lose the speed-up PowerShell gets. | CF-CMP-002 | none (STATIC-ONLY; would change golden samples of the affected CFs once the oracle exists) | LSN-TIER-04 (02-language/TIERING-POLICY.md) | diff --git a/docs-out/CONTRADICTIONS.md b/docs-out/CONTRADICTIONS.md index 88249dc..3cb0f36 100644 --- a/docs-out/CONTRADICTIONS.md +++ b/docs-out/CONTRADICTIONS.md @@ -5,3 +5,12 @@ Rows are never resolved by silent rewording. Status: `OPEN | RESOLVED`. Final st | CON-ID | statement A + location | statement B + location | evidence classes | status | resolution + evidence | files changed | |---|---|---|---|---|---|---| | CON-001 | `build.psm1::New-PSOptions` help text `.PARAMETER Framework`: "Defaults to 'net11.0'" (`build.psm1@L1230-L1260`) | same function: `[ValidateSet("net10.0")] [string]$Framework = "net10.0"` (`build.psm1@L1230-L1400`) and `PowerShell.Common.props` `TargetFramework` `net10.0` | E1 comment vs E1 code | RESOLVED | rule 1: code beats comments; framework is `net10.0` | docs-out/01-repository/build-system.md (noted in section 17) | +| CON-002 | `token.cs::TokenKind` XML doc comments mark `AndAnd`/`OrOr` as "(unimplemented)" (`src/System.Management.Automation/engine/parser/token.cs@L158-L161`) | `Tokenizer.NextToken` returns `AndAnd`/`OrOr` for `&&`/`||` in every mode (`src/System.Management.Automation/engine/parser/tokenizer.cs@L4612-L5081`) and the parser consumes them (`PipelineChainRule`) | E1 comment vs E1 code | RESOLVED | rule 1: code beats comments; the doc comment is stale. | docs-out/02-language/02.1-tokenizer.md (§6.1) | +| CON-003 | `TokenKind.Bxor` doc comment says `-xor` (`src/System.Management.Automation/engine/parser/token.cs@L256-L258`) | `s_tokenText[Bxor]` is `-bxor` and the operator table maps `bxor` to `Bxor` (`tokenizer.cs@L657-L693`) | E1 comment vs E1 code | RESOLVED | code beats comments; typo in comment. | docs-out/02-language/TOKEN-CATALOG.md | +| CON-004 | `Parser.cs` grammar comments: ternary branches are `ternary-expression`; `throw` takes `pipeline:opt` (`src/System.Management.Automation/engine/parser/Parser.cs@L6705-L6706`, `@L2288-L2289`) | code: `ExpressionRule` parses full `ExpressionRule` for both branches (`@L6696-L6790`); `ThrowStatementRule` calls `PipelineChainRule` (`@L2286-L2297`) | E1 comment vs E1 code | RESOLVED | rule 1: code beats comments; grammar in `GRAMMAR.ebnf.md` follows the code and flags [≠G]. | docs-out/02-language/GRAMMAR.ebnf.md | +| CON-005 | `StatementRule` XML doc: never returns null / returns an empty pipeline (`src/System.Management.Automation/engine/parser/Parser.cs@L1944-L1950`) | code returns `null` at end of input (`@L2099-L2111`) | E1 comment vs E1 code | RESOLVED | code beats comments; `StatementListRule` stops on null. | docs-out/02-language/02.2-parser.md (§9) | +| CON-006 | `ParserEventSource.MaxScriptLengthToLog = 50` (`src/System.Management.Automation/engine/parser/Parser.cs@L8252`) | `GetFileOrScript` logs up to 256 characters (`@L8274-L8277`); constant not referenced in `Parser.cs` | E1 constant vs E1 code | RESOLVED | helper behaviour used in the document; constant is unused in this file (other files unchecked: `UNVERIFIED`). | docs-out/02-language/02.2-parser.md (§15) | +| CON-007 | `ast.cs` header comment: private/internal methods live in `AstCompile.cs` (`src/System.Management.Automation/engine/parser/ast.cs@L5-L6`) | no such file exists in `engine/parser/`; compile-time pieces are in `Compiler.cs` and in `ast.cs` itself | E1 comment vs E1 tree | RESOLVED | tree listing wins; the comment is stale. | docs-out/02-language/02.3-ast.md (§9) | +| CON-008 | `SafeValues.cs` comment: `$OFS` cannot be used, a space is assigned (`src/System.Management.Automation/engine/parser/SafeValues.cs@L579-L581`) | code reads `$OFS` from the session state when a context exists, default space otherwise (`@L546-L556`) | E1 comment vs E1 code | RESOLVED | code beats comments. | docs-out/02-language/02.3-ast.md (§7.2) | +| CON-009 | `Compiler.cs` comment near `DefinePowerShellTypes`: class-definition errors should be found at parse time, the compile-time check is a "short term solution" (`src/System.Management.Automation/engine/parser/Compiler.cs@L2717-L2745`) | `TypeDefiner.DefineTypes` reports duplicate/unknown/invalid-member errors only when the root script block is compiled (`src/System.Management.Automation/engine/parser/PSType.cs@L1335-L1431`) | E1 comment vs E1 code | RESOLVED | code is authoritative; documented as a stage (02.4-semantic-analysis.md §6). | docs-out/02-language/02.4-semantic-analysis.md | +| CON-010 | study prompt hint: `engine/lang/parserutils.cs` holds parser utilities | file holds runtime operator helpers (`ParserOps`), flow-control exceptions and `InterpreterError` (`src/System.Management.Automation/engine/lang/parserutils.cs@L23-L2010`) | prompt hint vs E1 | RESOLVED | source wins; documented as runtime helpers. | docs-out/02-language/02.5-compiler-and-engine.md | diff --git a/docs-out/DECISIONS.md b/docs-out/DECISIONS.md index 4002203..778e2f9 100644 --- a/docs-out/DECISIONS.md +++ b/docs-out/DECISIONS.md @@ -13,3 +13,10 @@ | D-009 | The interpreter lives in `src/System.Management.Automation/engine/interpreter/` (38 files). | Closes hint C4 of Appendix C; confirmed by directory listing. | | D-010 | Documents are committed to branch `claude/repo-review-explanation-22f3xr` per phase; no PR. | Owner answer in session. | | D-011 | `DotnetRuntimeMetadata.json` channel `9.0.1xx-preview6` is inconsistent with `sdkImageVersion` `10.0.403`; `global.json` + `PowerShell.Common.props` (net10.0) are taken as authoritative for the pin. | Observed in the file; left unresolved because the file is only build metadata (see external-dependencies.md). | +| D-012 | `consist.py validate-refs` blocks on non-docs changes after the pin; the only post-pin changes are `.github/workflows/*` and the root `Lite Master Prompt.md` (dependabot merges and a prompt upload); `src/` and `test/` are identical to the pin (`git diff 523031e HEAD -- src test` empty). The gate was run with the pin check bypassed in a throwaway interpreter, not by editing the tool: 496 refs, 0 failures. | Coordinator should either relax the gate to `src test` or record the upstream merges. | +| D-013 | `data/tables.json` entries for `token.cs::s_tokenText` and `::s_staticTokenFlags` (and `s_keywordText` neighbours) contain text running past the array's closing `};` (extractor end line too late for `s_tokenText`: L959-L1211 vs array end ~L1157). `tools/gen_T21.py` cuts at the first `};` and drops `#region` lines; lengths are asserted against the `TokenKind` range. | Dataset defect for the coordinator: fix `p0_extract2.py` table end detection. | +| D-014 | This run uses `PowerShell-Architecture-Analysis-Prompt-v4.md` (full study) as the governing prompt, not `Lite Master Prompt.md`, because Phase 0-1 were produced under v4 (D-001..D-011) and `PROGRESS.md` follows its task IDs. The Lite prompt was added later (commit 8f35650) and is not applied. | Owner may switch; doing so reshapes the output tree. | +| D-015 | CF-GRM-001 (reserved owner `GRAMMAR.ebnf.md`) is stated in `GRAMMAR.ebnf.md` §3a; `02.2-parser.md` links it. CF-ERR-002 is stated in `ERROR-CATALOG.md`. `OPERATOR-TABLE.md` (CF-OP-001) is T28's; the parser document carries only a generated precedence view and links to it. | Keeps one statement per fact (rule 5). | +| D-016 | `ERROR-CATALOG.md` covers `ParserStrings.resx` only (412 keys); semantic-check keys in that resx are listed in section 4 but explained in T24. | Scope split between T22 and T24. | +| D-017 | `AST-CATALOG.md` children column is extracted from `InternalVisit`/`InternalVisitChildren` bodies (receivers of `.InternalVisit(`, loop variables mapped to their collection); the dataset's `ast_typed_members` is not used because it misses tuple and collection children. Statistics: 76 nodes in `ast.cs`, 9 concrete leaf-looking rows (checked: leaves or children visited through `InternalVisitChildren`). | Source reading for `IfStatementAst`, `ScriptBlockAst`, `ErrorStatementAst`, `InvokeMemberExpressionAst`. | +| D-018 | T24, T25 and T27 were authored in one batch (inline, no subagents). Drafts use `[[file::Type.Member]]` tokens expanded by `tools/expand_refs.py` from `data/types.json` and `data/members-tierA.json`, so no reference range is typed by hand; unresolved tokens fail the expansion. The T27 documents own CF-CMP-001/002/003; `AlwaysCompile` is documented as unused after `rg` over `engine/`. | Rule 1 (evidence) and rule 9 (generated catalogs). | diff --git a/docs-out/PROGRESS.md b/docs-out/PROGRESS.md index d30d646..cfba2cb 100644 --- a/docs-out/PROGRESS.md +++ b/docs-out/PROGRESS.md @@ -7,8 +7,8 @@ Run mode: STATIC-ONLY (`ALLOW_STATIC_ONLY=true`). Branch `claude/repo-review-exp |---|---|---| | Preflight (section 2) | done | `.work/environment.md`, `.work/pin.json`, DECISIONS D-001..D-011 | | 0 Recon, pinning, tooling | done | datasets in `data/`, tools in `tools/`, registers, glossary, census, dependency graph, source-tree map, 297 folder cards + 40 project cards (generated; annotation overlay pending) | -| 1 Repository map | todo | layered-architecture, build-system, ci-cd-and-packaging, coding-standards | -| 2 Language pipeline | in progress | wave 1: T21 T22 T23 T24 T25 T27 | +| 1 Repository map | done | layered-architecture, build-system, ci-cd-and-packaging, coding-standards (committed 4a6efaf) | +| 2 Language pipeline | in progress | wave 1: T21 DONE (draft); T22 DONE (draft); T23 DONE (draft); T24 T25 T27 DONE (draft); T26 (binders, runtime operations), T28, T29, T210, T211 and the coordinator documents of 02-language are next | | 3 Runtime | todo | | | 4 Hosting/SDK/remoting/platform/command modules | todo | | | 5 Security | todo | | @@ -21,4 +21,13 @@ Run mode: STATIC-ONLY (`ALLOW_STATIC_ONLY=true`). Branch `claude/repo-review-exp T21 tokenizer | T22 parser/grammar/errors | T23 AST | T24 semantic | T25 compiler/IR | T26 binders+runtime ops | T27 interpreter+tiering | T28 types/operators/conversion | T29 scope/variables+execution context | T210 language features+HOWTO | T211 tooling | T31..T39 runtime subsystems (see `.work/DOC-MAP.md`). ## Document checklist -Maintained as agents report; see `.work/reports/`. +| Task | Document | Status | +|---|---|---| +| T21 | `02-language/02.1-tokenizer.md`, `02-language/TOKEN-CATALOG.md` (generated by `tools/gen_T21.py`) | draft written; checks pass: render-canonical, check-claims, lint-terms, check-front-matter, numbers lint, validate-refs (pin gate bypassed, D-012) | +| T22 | `02-language/02.2-parser.md`, `GRAMMAR.ebnf.md`, `ERROR-CATALOG.md` (generated parts by `tools/gen_T22.py`) | draft written; CF-GRM-001, CF-ERR-002, DEV-GRM-010; all checks pass (validate-refs with pin gate bypassed, D-012) | +| T23 | `02-language/02.3-ast.md`, `AST-CATALOG.md` (generated by `tools/gen_T23.py`) | draft written; DEV-AST-020, CON-007/008, VB-008..012; all checks pass (validate-refs with pin gate bypassed, D-012) | +| T24 | `02-language/02.4-semantic-analysis.md` (check table generated by `tools/gen_T24.py`: 133 sites, 104 error IDs) | draft written; DEV-SEM-030, CON-009; checks pass | +| T25 | `02-language/02.5-compiler-and-engine.md`, `COMPILATION-MAP.md` (`tools/gen_T25.py` + `tools/annotations/T25-compilation-map.json`), `IR-SPEC.md` | draft written; DEV-ENG-040, CON-010, VB-015; checks pass | +| T27 | `02-language/INSTRUCTION-SET.md` (`tools/gen_T27.py`), `TIERING-POLICY.md` (owns CF-CMP-001/002/003), `EQUIVALENCE-SPEC.md` | draft written; DEV-TIER-060, DEV-EQV-061, VB-013/014/016; checks pass | + +Next step: T26 (binders, runtime operations), T28 (types, operators, conversions), T29 (scope and variables), T210/T211, then the coordinator documents (pipeline map, stage contracts, lifecycle). Open: CF-OP-001 (OPERATOR-TABLE, T28) is linked from the parser doc but not yet written. diff --git a/docs-out/VERIFICATION-BACKLOG.md b/docs-out/VERIFICATION-BACKLOG.md index 2314df9..f0067e8 100644 --- a/docs-out/VERIFICATION-BACKLOG.md +++ b/docs-out/VERIFICATION-BACKLOG.md @@ -5,3 +5,18 @@ Every `UNVERIFIED` claim and every `BLOCKED` item. Under STATIC-ONLY (owner deci | ID | status | scope | description | owner-actionable fix | impact | |---|---|---|---|---|---| | VB-001 | BLOCKED | all `CORE` behavior claims, golden samples, equivalence across execution modes, Roslyn-based counts | No .NET SDK (global.json requires 10.0.403) and no `pwsh` on the study host; the oracle (5.4) could not run. | Install .NET SDK 10.0.403 and `pwsh` 7.6.x on a host, run `tools/oracle-harness` (to be authored) in a scratch copy, then rerun the study with `ALLOW_STATIC_ONLY=false` (Phase 8 + gates 9, 10). | Behavior statements stay `FACT-SRC`/`INFERENCE`; acceptance status capped at CONDITIONAL. | +| VB-002 | OPEN | LANG-TOKENIZER-001 §7.1 | Behavior of a bare `0x` (no digits): scanner prepends `0` to the digit string, so the emptiness test cannot fire (`tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163`). | Run `pwsh -c '0x'` and tokenizer API on the host (`UNVERIFIED`). | Number-literal edge case. | +| VB-003 | OPEN | LANG-TOKENIZER-001 §7.4 | `#requires` match does not check the character after `requires`, so `#requiresX` may be treated as a requires line. | Run `[System.Management.Automation.Language.Parser]::ParseInput` on `#requiresfoo` and inspect `ScriptRequirements`. | Pragma edge case. | +| VB-004 | OPEN | LANG-TOKENIZER-001 §7.2 | `$(...)` inside a string is pre-scanned by counting parentheses only; a `)` inside a quoted string within it may end the scan early. | Run `"$( ')' )"` with `pwsh`; also search `test/powershell/Language/Parser` for an asserting test. | String-interpolation edge case. | +| VB-005 | OPEN | LANG-TOKENIZER-001 §16 | Per-assertion mapping from `test/powershell/Language/Parser/*` to tokenizer rules not done. | Pester inventory pass for tokenizer-specific `Describe/Context/It` names. | Test citations. | +| VB-006 | OPEN | LANG-PARSER-001 §9 | Exact behavior of error recovery for common mistakes (missing `}`, missing `)`, stray tokens): messages, order, and AST shape. | Run `Parser.ParseInput` on a corpus of broken scripts with `pwsh` and diff error IDs/extents against the catalog. | Error recovery fidelity. | +| VB-007 | OPEN | LANG-GRAMMAR-001 §7 | `for` loop with newline-only separators (no `;`): code consumes `Semi` via `PeekToken`; confirm whether newline between initializer/condition/iterator is accepted. | Parse `for ($i=0\n$i -lt 3\n$i++) {}` with `pwsh`. | Grammar edge case. | +| VB-008 | OPEN | LANG-AST-001 §9 | `UsingStatementAst.Copy` chooses its constructor by `Alias != null` only; module-spec forms may be lost or throw (`ast.cs@L3037-L3045`). | `ParseInput('using module @{ModuleName="x";ModuleVersion="1.0"}')` then `.Copy()` with pwsh. | Possible defect, tooling only. | +| VB-009 | OPEN | LANG-AST-001 §9 | `ConfigurationDefinitionAst.InternalVisit` discards the action returned by `Body.InternalVisit` (`ast.cs@L6599-L6620`). | Visitor that returns `StopVisit` inside a configuration body; check post-action. | DSC only (out of scope). | +| VB-010 | OPEN | LANG-AST-001 §9 | `BaseCtorInvokeMemberExpressionAst` has no `Copy` override; copy may degrade to `InvokeMemberExpressionAst`. | Copy a class constructor's AST and inspect node type. | Class feature (EXTENDED). | +| VB-011 | OPEN | LANG-AST-001 §9 | `ForEachStatementAst.ThrottleLimit` and `DataStatementAst.CommandsAllowed` are not visited by `InternalVisit`. | `FindAll` for the throttle-limit expression in a `foreach -parallel -throttlelimit 3` node. | Search/tooling completeness. | +| VB-012 | OPEN | LANG-AST-001 §9 | Multi-line `ScriptExtent.Text` uses the 1-based column as a 0-based index for the first line (`Position.cs@L696-L815`). | Construct a multi-line `ScriptExtent` and print `.Text`. | Public-API edge case. | +| VB-013 | OPEN | LANG-TIERING-001 §9 | `LightCompiler` throws `NotImplementedException` for unsupported IR shapes and no handler was found in `Compiler.CompileTree`; the compiler is believed never to emit them. | Search `Compiler.cs` for each unsupported node kind; run a corpus with `pwsh` under a debugger to confirm no interpreter exception. | Engine robustness. | +| VB-014 | OPEN | LANG-TIERING-001 §16, LANG-EQUIV-001 §16 | No test located that exercises interpreter/JIT parity or loop promotion. | `rg` the xUnit and Pester trees for `CompileSynchronously`, `EnterLoop`, large-loop scripts. | Testability. | +| VB-015 | OPEN | LANG-ENGINE-001 §19 | `?.` null-conditional wrapper contains the target expression twice (`Compiler.cs::GetNullConditionalWrappedExpression`); a side-effecting target may be evaluated twice. | Run `(f)?.Length` with a function that logs, in `pwsh`. | Possible semantic bug. | +| VB-016 | OPEN | LANG-EQUIV-001 §7.1 | Typed arithmetic/comparison instruction defects (UInt64 cast to Int16, unsigned mapped to signed, `Equal(string)`) are believed unreachable. | Confirm no code path emits typed non-double arithmetic; run unsigned arithmetic in a loop under the interpreter. | Latent interpreter defects. | diff --git a/docs-out/data/canonical-facts.json b/docs-out/data/canonical-facts.json index f08a005..7cbbfea 100644 --- a/docs-out/data/canonical-facts.json +++ b/docs-out/data/canonical-facts.json @@ -11,29 +11,47 @@ { "id": "CF-CMP-001", "subject": "compile-versus-interpret decision rule", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Compile-versus-interpret rule.** Every named block is turned into one expression-tree lambda. At creation of the delegate (`Compiler.CompileTree`), the tiering mode is `NeverCompile` if the script block's total statement count (`_stmtCount`) is greater than 300, otherwise `CompileOnDemand`; `AlwaysCompile` exists but no caller passes it. The delegate starts as an **interpreted** `LightLambda` with a call counter of 32 (`CompileOnDemand`, threshold -1 mapped to `DefaultCompilationThreshold`) or `int.MaxValue` (`NeverCompile`). Each call decrements the counter; the call that reads 0 (the 33rd) queues JIT compilation of the whole lambda on a thread-pool thread and is itself still interpreted; once the compiled delegate exists, every later call uses it. A lambda is **never interpreted** (JIT-compiled at once, synchronously) when the light compiler meets a method call with a by-ref parameter, an instance call on a non-primitive value type, or a constructor with a by-ref parameter; and trap-handler lambdas and single-expression lambdas are always built with `LambdaExpression.Compile()` (never interpreted). Loops are promoted separately (CF-CMP-002).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L1974-L2077", + "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2096", + "src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052", + "src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246", + "src/System.Management.Automation/engine/interpreter/LightDelegateCreator.cs::LightDelegateCreator@L30-L220" + ], "owner": "02-language/TIERING-POLICY.md", - "status": "open" + "status": "open", + "_task": "T27" }, { "id": "CF-CMP-002", "subject": "every threshold and limit of the tiering policy", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Tiering thresholds and limits.** Statement count above which a script block is never JIT-compiled by call count: 300 (`_stmtCount > 300`, `Compiler.Compile`). Interpreted calls before JIT: 32 (`LightCompiler.DefaultCompilationThreshold`), so JIT starts on the 33rd call; `int.MaxValue` for `NeverCompile`; threshold 0 or 1 means synchronous compilation (never produced by `Compiler`). Loop iterations before loop JIT: 16 (constructor argument of `EnterLoopInstruction` in `EnterLoopExpression.AddInstructions`), so the loop JIT starts on the 17th arrival at the loop top. Loop statement count at or above which a loop is never promoted: 300 (`LoopStatementCount < 300` required to emit the loop counter). Maximum delegate parameters of an interpreted lambda: 16 (`LightLambda.MaxParameters`). Compile helper cache of delegate `Run` factories: 100 entries (`CacheDict` in `LightLambda`). Parsed-script cache (shares compiled delegates between clones): cleared completely when it holds more than 1024 entries (`ScriptBlock.CacheScriptBlock`).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7021", + "src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052", + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177" + ], "owner": "02-language/TIERING-POLICY.md", - "status": "open" + "status": "open", + "_task": "T27" }, { "id": "CF-CMP-003", "subject": "optimized-versus-unoptimized preconditions", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Optimized versus unoptimized.** A script block clause is run from the optimized code (variables in fixed tuple slots) only if the engine is not debugging (`ExecutionContext._debuggingMode <= 0`) **and** the invocation creates a new local scope (`createLocalScope`, i.e. not dot-sourced); the same two conditions are used for script cmdlets. If the optimized compile shows that any slot-allocated variable name could be an `AllScope` variable (`VariableAnalysis.AnyVariablesCouldBeAllScope`), the unoptimized variant is used instead, and both variants are compiled. In the unoptimized variant only the automatic variables (plus the preference variables for script cmdlets) get tuple slots and every other variable is looked up by name. Independently of this choice, variable analysis allocates no slots for a block that uses `New-Variable`, `Remove-Variable`, `Set-Variable`, `Set-PSBreakpoint` (or their aliases), dot-sourcing or `$private:` variables.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83", + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177", + "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::PSScriptCmdlet@L2179-L2552", + "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847" + ], "owner": "02-language/TIERING-POLICY.md", - "status": "open" + "status": "open", + "_task": "T27" }, { "id": "CF-CNV-001", @@ -65,11 +83,17 @@ { "id": "CF-ERR-002", "subject": "parse-error limits and recovery rule", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Parse-error limits and recovery rule.** The parser never throws for a syntax error: each error is appended to `Parser.ErrorList` after de-duplication (an error is dropped only if an earlier error has the same ID and the same start and end line and column), there is no limit on the number of errors, and parsing continues by returning partial or error nodes (`ErrorStatementAst`, `ErrorExpressionAst`), by pretending a missing closing token was present, or by skipping tokens to a synchronization token (`SyncOnError`, which tracks nesting of `()`, `{}` and `[]` and never consumes end of input); `ParseError.IncompleteInput` is true only when the error was reported with `ReportIncompleteInput` and the error position is at the end of the script apart from whitespace and comments. The only exceptions are an exhausted execution stack (the whole parse is retried once on a new thread, then `ScriptTooComplicated` is reported) and any other exception during `ParseFile`/`ParseInput`, which is wrapped as a `ParseException` with `UnrecoverableParserError`.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.SyncOnError@L366-L432", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ReportIncompleteInput@L8111-L8141" + ], "owner": "02-language/ERROR-CATALOG.md", - "status": "open" + "status": "open", + "_task": "T22" }, { "id": "CF-ETS-001", @@ -92,11 +116,17 @@ { "id": "CF-GRM-001", "subject": "statement, pipeline and command grammar; statement terminators", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Statement and terminator rule.** A statement list is parsed as: skip any run of newlines and semicolons; parse a statement; skip any run of newlines and semicolons; stop when the next token is `)` or `}` or no statement can be parsed. A terminator is **not** required between statements that end with a closing `}` (e.g. `if`, loops, `try`, `function`), because the list only skips optional terminators; a pipeline (or any statement parsed by `PipelineChainRule`) ends at newline, `;`, `)`, `}`, `&&`/`||` (chain), `&` (background) or end of input, and any other token after a complete pipeline element is `UnexpectedToken`. Newlines continue a construct only in these places: after `|`, before a `|` that starts the next line, after `&&` and `||`, after each binary or unary operator and after `,`, after `=` in an assignment, inside `( )`, after `?`/`:` of a ternary, between `}` and `elseif`/`else`/`catch`/`finally`, and after a backtick line continuation handled by the tokenizer.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineRule@L5990-L6160", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.CatchBlockRule@L5494-L5591" + ], "owner": "02-language/GRAMMAR.ebnf.md", - "status": "open" + "status": "open", + "_task": "T22" }, { "id": "CF-LCY-001", @@ -110,20 +140,31 @@ { "id": "CF-LEX-001", "subject": "tokenizer mode switching rule (command vs expression mode, when it switches)", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Mode rule.** The tokenizer starts in Command mode; the parser switches to Expression mode when it parses an expression operand/operator position (including the first element of a pipeline when it is not a command), to Command mode when it parses a command's arguments or the body of `{ }`, `( )`, `$( )`, to TypeName mode inside `[...]` type literals, and to Signature mode for class/enum/method headers; every switch is restored in a `finally` block, and a switch discards and rescans a pending lookahead token unless its kind is `ParseModeInvariant`.", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535", + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.CommandRule@L6444-L6684", + "src/System.Management.Automation/engine/parser/Parser.cs::Parser.ExpressionRule@L6696-L6790" + ], "owner": "02-language/02.1-tokenizer.md", - "status": "open" + "status": "open", + "_task": "T21" }, { "id": "CF-LEX-002", "subject": "numeric-literal rules (radix, suffixes, multipliers)", - "statement": "", - "label": "UNVERIFIED", - "evidence": [], + "statement": "**Numeric literal rules.** A number is an optional sign (`-`, en/em dash/horizontal bar, or `+`; only when `AllowSignedNumbers`), then either `.digits[exponent]`, `0x` hex digits, `0b` binary digits, or decimal digits with optional `.digits` and `e/E[+|-]digits`; then an optional type suffix (`u s l d y n`, plus a second character only after `u`: `ul us uy`), then an optional multiplier `k m g t p` that **must be followed by `b`/`B`** (powers of 1024; the suffix comes before the multiplier); the literal must be followed by a token terminator (in Expression mode also by a binary-operator start character) or the text is not a number and is rescanned as a generic token. Value typing: no suffix \u2192 `int`, else `long`, else (decimal format only) `decimal`, else `double`; real literals \u2192 `double` unless suffix `d` (`decimal`); hex and binary are two's complement with the sign decided by the top digit (`0xFFFFFFFF` \u2192 `-1`, `0xFF` \u2192 `255`), a 16-hex-digit or 64-bit-digit literal without `u` becomes `long`; the multiplier is applied to the value before the range cast and failure to fit returns `false` (reported as `BadNumericConstant` in Expression mode, otherwise the text becomes a generic token).", + "label": "FACT-SRC", + "evidence": [ + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163", + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882", + "src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumber@L3884-L3931" + ], "owner": "02-language/02.1-tokenizer.md", - "status": "open" + "status": "open", + "_task": "T21" }, { "id": "CF-MOD-001", diff --git a/docs-out/data/claims/T21.jsonl b/docs-out/data/claims/T21.jsonl new file mode 100644 index 0000000..41e3d46 --- /dev/null +++ b/docs-out/data/claims/T21.jsonl @@ -0,0 +1,19 @@ +{"id": "CLM-LEX-0001", "subject_key": "tokenizer.mode.initial", "predicate": "initial mode after Initialize", "value": "Command", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-001"} +{"id": "CLM-LEX-0002", "subject_key": "tokenizer.mode.keywords_recognized_in", "predicate": "keywords recognized only in modes", "value": "Command,Signature (and not WantSimpleName)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanIdentifier@L4402-L4458"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0003", "subject_key": "tokenizer.mode.dash_operators_recognized_in", "predicate": "-eq/-and style operators produced only in mode", "value": "Expression", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanParameter@L3185-L3338"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-001"} +{"id": "CLM-LEX-0004", "subject_key": "tokenizer.mode.switch_rescans_lookahead", "predicate": "mode switch rescans peeked token unless ParseModeInvariant", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-001"} +{"id": "CLM-LEX-0005", "subject_key": "tokenizer.lookahead.depth", "predicate": "parser token lookahead", "value": "1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser@L495-L497"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0006", "subject_key": "tokenizer.numeric.multiplier_requires_b", "predicate": "multiplier suffix k/m/g/t/p must be followed by b", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0007", "subject_key": "tokenizer.numeric.multiplier_base", "predicate": "multiplier base", "value": "1024", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0008", "subject_key": "tokenizer.numeric.type_suffixes", "predicate": "single type suffix letters", "value": "u s l d y n", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanNumberHelper@L3933-L4163"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0009", "subject_key": "tokenizer.numeric.signed_requires_flag", "predicate": "leading sign scanned as part of a number only when AllowSignedNumbers", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0010", "subject_key": "tokenizer.numeric.default_promotion", "predicate": "no-suffix integer promotion", "value": "int,long,decimal(decimal format only),double", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882"], "doc": "LANG-TOKENIZER-001", "cf": "CF-LEX-002"} +{"id": "CLM-LEX-0011", "subject_key": "tokenizer.string.single_quote_escape", "predicate": "backtick escapes in single-quoted strings", "value": "none", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanStringLiteral@L2317-L2360"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0012", "subject_key": "tokenizer.herestring.footer", "predicate": "here-string footer position", "value": "quote+@ at start of a line", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanHereStringLiteral@L2685-L2753"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0013", "subject_key": "tokenizer.subexpr.prescan", "predicate": "$() in strings pre-scanned by counting parentheses only", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanSubExpression@L2362-L2447"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0014", "subject_key": "tokenizer.line_endings", "predicate": "characters that end a line for position tracking", "value": "CR, LF, CRLF", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Initialize@L762-L797"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0015", "subject_key": "tokenizer.comments.block_nest", "predicate": "block comments nest", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanBlockComment@L1809-L1837"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0016", "subject_key": "tokenizer.redirection.stream_digits", "predicate": "digits forming stream-numbered redirection", "value": "1-6", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0017", "subject_key": "tokenizer.chain_operators.mode_dependent", "predicate": "&& and || tokenized in every mode", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0018", "subject_key": "tokenizer.questionmark.expression_only", "predicate": "?, ??, ??= tokens produced only in Expression mode", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.NextToken@L4612-L5081"], "doc": "LANG-TOKENIZER-001", "cf": null} +{"id": "CLM-LEX-0019", "subject_key": "tokenizer.requires.recognition", "predicate": "#requires must follow '#' immediately", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanToEndOfCommentLine@L1605-L1754"], "doc": "LANG-TOKENIZER-001", "cf": null} diff --git a/docs-out/data/claims/T22.jsonl b/docs-out/data/claims/T22.jsonl new file mode 100644 index 0000000..afb8e64 --- /dev/null +++ b/docs-out/data/claims/T22.jsonl @@ -0,0 +1,17 @@ +{"id": "CLM-GRM-0001", "subject_key": "parser.lookahead.tokens", "predicate": "maximum pushed-back tokens", "value": "1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser@L495-L497"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0002", "subject_key": "parser.binary_operators.associativity", "predicate": "associativity of all binary operators", "value": "left", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0003", "subject_key": "parser.comma.vs_binary", "predicate": "comma operator binds tighter than binary operators (operands are ArrayLiteralRule)", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0004", "subject_key": "parser.statement.start", "predicate": "statement start tries expression in Expression mode before command", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0005", "subject_key": "parser.statement.terminator_after_block", "predicate": "terminator required after statement ending in closing brace", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0006", "subject_key": "parser.assignment.rhs", "predicate": "right side of an assignment is parsed by", "value": "StatementRule", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0007", "subject_key": "parser.hashtable.value", "predicate": "hashtable entry value is parsed by", "value": "StatementRule in Command mode", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.GetKeyValuePair@L7405-L7491"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0008", "subject_key": "parser.chain_operators.associativity", "predicate": "&& and || chaining", "value": "left nested PipelineChainAst", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988"], "doc": "LANG-PARSER-001", "cf": "CF-GRM-001"} +{"id": "CLM-GRM-0009", "subject_key": "parser.errors.dedup_key", "predicate": "error de-duplication key", "value": "same ErrorId and same start/end line and column", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057"], "doc": "LANG-PARSER-001", "cf": "CF-ERR-002"} +{"id": "CLM-GRM-0010", "subject_key": "parser.errors.max_count", "predicate": "limit on number of parse errors", "value": "none", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SaveError@L8026-L8057"], "doc": "LANG-PARSER-001", "cf": "CF-ERR-002"} +{"id": "CLM-GRM-0011", "subject_key": "parser.stack.retry", "predicate": "stack exhaustion handling", "value": "retry once on a new thread then ScriptTooComplicated", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227"], "doc": "LANG-PARSER-001", "cf": "CF-ERR-002"} +{"id": "CLM-GRM-0012", "subject_key": "parser.array_type.max_dimensions", "predicate": "maximum array type rank", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.CompleteArrayTypeName@L1523-L1627"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0013", "subject_key": "parser.using_assembly.resolution_time", "predicate": "using assembly path resolved at", "value": "parse time", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0014", "subject_key": "parser.class.constrained_language", "predicate": "class keyword in Constrained Language Mode", "value": "rejected unless lockdown policy is Audit", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ClassDefinitionRule@L4208-L4379"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0015", "subject_key": "parser.mode_analysis.post_parse_checks", "predicate": "PerformPostParseChecks runs", "value": "only when ParseMode is Default", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0016", "subject_key": "parser.switch.flags_matching", "predicate": "switch flags -regex -wildcard -exact -casesensitive -parallel -file matched by", "value": "case-insensitive prefix", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.SwitchStatementRule@L2543-L2891"], "doc": "LANG-PARSER-001", "cf": null} +{"id": "CLM-GRM-0017", "subject_key": "parser.member_access.whitespace", "predicate": "whitespace allowed before member access operator or invocation paren", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Parser.cs::Parser.MemberAccessRule@L7718-L7775"], "doc": "LANG-PARSER-001", "cf": null} diff --git a/docs-out/data/claims/T23.jsonl b/docs-out/data/claims/T23.jsonl new file mode 100644 index 0000000..69ae374 --- /dev/null +++ b/docs-out/data/claims/T23.jsonl @@ -0,0 +1,13 @@ +{"id": "CLM-AST-0001", "subject_key": "ast.parent.set_once", "predicate": "a node can have a parent assigned more than once", "value": "false (AstIsReused)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0002", "subject_key": "ast.visit.pre_order_post_hook", "predicate": "PostVisit is called on every exit path (Continue, SkipChildren, StopVisit)", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0003", "subject_key": "ast.visit.v2_nodes", "predicate": "nodes visited only by second-generation visitors", "value": "TypeDefinition, PropertyMember, FunctionMember, BaseCtorInvokeMemberExpression, UsingStatement, ConfigurationDefinition, DynamicKeywordStatement, TernaryExpression, PipelineChain", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0004", "subject_key": "ast.find.root_tested", "predicate": "Find/FindAll test the root node itself", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/AstVisitor.cs::AstSearcher@L404-L617"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0005", "subject_key": "ast.find.nested_skip", "predicate": "nested scopes skipped when searchNestedScriptBlocks is false", "value": "FunctionDefinition, Trap, ScriptBlockExpression (not ScriptBlockAst)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/AstVisitor.cs::AstSearcher@L404-L617"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0006", "subject_key": "ast.safevalue.max_visits", "predicate": "safe-value visit cap", "value": "5000", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0007", "subject_key": "ast.safevalue.max_hashtable_keys", "predicate": "safe-value hashtable key cap", "value": "500", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0008", "subject_key": "ast.safevalue.binary_expression", "predicate": "binary expressions are safe values", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0009", "subject_key": "ast.constant.evaluation", "predicate": "constant folding value computed by", "value": "compiling and invoking the expression", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ConstantValues.cs::ConstantValueVisitor@L332-L577"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0010", "subject_key": "ast.hashtable.value_type", "predicate": "hashtable entry value node type", "value": "StatementAst", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::HashtableAst@L10128-L10218"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0011", "subject_key": "ast.assignment.rhs_normalization", "predicate": "single-command-expression pipeline on the right of an assignment is replaced by the expression", "value": "true", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::AssignmentStatementAst@L6379-L6495"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0012", "subject_key": "ast.pipelinechain.shape", "predicate": "PipelineChainAst right side", "value": "always PipelineAst (left nested)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::PipelineChainAst@L5549-L5647"], "doc": "LANG-AST-001", "cf": null} +{"id": "CLM-AST-0013", "subject_key": "ast.ternary.visitor_gate", "predicate": "TernaryExpressionAst visited by v1 visitor", "value": "children yes, VisitTernaryExpression no", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::TernaryExpressionAst@L7404-L7497"], "doc": "LANG-AST-001", "cf": null} diff --git a/docs-out/data/claims/T24.jsonl b/docs-out/data/claims/T24.jsonl new file mode 100644 index 0000000..2ed5458 --- /dev/null +++ b/docs-out/data/claims/T24.jsonl @@ -0,0 +1,9 @@ +{"id": "CLM-SEM-0001", "subject_key": "semantic.order", "predicate": "post-parse analysis order", "value": "SymbolResolver then SemanticChecks (ParseMode.Default only)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/ast.cs::ScriptBlockAst.PerformPostParseChecks@L1416-L1440"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0002", "subject_key": "semantic.unresolved_type_error", "predicate": "unresolved type outside a type definition", "value": "no parse error (late bound)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SymbolResolver.cs::SymbolResolver@L275-L729"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0003", "subject_key": "semantic.array_type.max_depth", "predicate": "array type name nesting limit", "value": "200 (ScriptTooComplicated)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0004", "subject_key": "variableanalysis.force_dynamic", "predicate": "slot sentinel for dynamic variable", "value": "-2", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0005", "subject_key": "variableanalysis.unanalyzed", "predicate": "slot sentinel for unanalyzed variable", "value": "-1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0006", "subject_key": "variableanalysis.pessimizing_cmdlets", "predicate": "commands that disable slot allocation", "value": "New-Variable, Remove-Variable, Set-Variable, Set-PSBreakpoint and aliases", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/VariableAnalysis.cs::FindAllVariablesVisitor@L44-L323"], "doc": "LANG-SEMANTIC-001", "cf": "CF-CMP-003"} +{"id": "CLM-SEM-0007", "subject_key": "typeresolver.order", "predicate": "type name resolution order", "value": "session scope types, builtin accelerators, assemblies, user accelerators, then using-namespace retries", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolver@L31-L506"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0008", "subject_key": "restricted.default_variables", "predicate": "variables allowed in data sections by default", "value": "PSCulture, PSUICulture, true, false, null", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454"], "doc": "LANG-SEMANTIC-001", "cf": null} +{"id": "CLM-SEM-0009", "subject_key": "typedefiner.run_time", "predicate": "class and enum definition errors reported at", "value": "compile of the root script block", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505"], "doc": "LANG-SEMANTIC-001", "cf": null} diff --git a/docs-out/data/claims/T25.jsonl b/docs-out/data/claims/T25.jsonl new file mode 100644 index 0000000..81965a1 --- /dev/null +++ b/docs-out/data/claims/T25.jsonl @@ -0,0 +1,9 @@ +{"id": "CLM-ENG-0001", "subject_key": "compiler.visit_method_count", "predicate": "Visit methods in Compiler", "value": "63", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0002", "subject_key": "compiler.statement_dispatch", "predicate": "statement lists with several statements use", "value": "integer dispatch with goto and per-statement catch", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0003", "subject_key": "compiler.return_semantics", "predicate": "return writes the value to", "value": "the output pipe (class methods: return pipe)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitReturnStatement@L5573-L5627"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0004", "subject_key": "compiler.switch.clauses", "predicate": "switch tests", "value": "every clause for every item; default only when none matched", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitSwitchStatement@L4410-L4542"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0005", "subject_key": "compiler.foreach.enumeration", "predicate": "foreach enumerator creation", "value": "PSEnumerableBinder; non-enumerable wrapped in one-element array", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateIteratorStatement@L4931-L5056"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0006", "subject_key": "compiler.loop.nonlocal_break", "predicate": "non-local break/continue", "value": "BreakException/ContinueException caught by enclosing loop", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateLoopBreakContinueCatchBlocks@L4731-L4758"], "doc": "LANG-ENGINE-001", "cf": null} +{"id": "CLM-ENG-0007", "subject_key": "compiler.scriptblock.cache_cap", "predicate": "parsed script block cache cap", "value": "1024 (cleared completely when exceeded)", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177"], "doc": "LANG-ENGINE-001", "cf": "CF-CMP-002"} +{"id": "CLM-ENG-0008", "subject_key": "compiler.single_expression.tier", "predicate": "single-expression and trap lambdas", "value": "always JIT compiled", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleExpression@L2204-L2227"], "doc": "LANG-ENGINE-001", "cf": "CF-CMP-001"} +{"id": "CLM-ENG-0009", "subject_key": "compiler.log_segment", "predicate": "script block log segment size when text length >= 20000", "value": "10000 plus random 0..9999", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177"], "doc": "LANG-ENGINE-001", "cf": null} diff --git a/docs-out/data/claims/T27.jsonl b/docs-out/data/claims/T27.jsonl new file mode 100644 index 0000000..810a50c --- /dev/null +++ b/docs-out/data/claims/T27.jsonl @@ -0,0 +1,10 @@ +{"id": "CLM-TIER-0001", "subject_key": "tiering.stmt_limit", "predicate": "statement count above which NeverCompile is chosen", "value": "300", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.Compile@L1974-L2077"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0002", "subject_key": "tiering.call_threshold", "predicate": "interpreted calls before JIT is queued (DefaultCompilationThreshold)", "value": "32", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0003", "subject_key": "tiering.loop_iterations", "predicate": "loop iterations before loop JIT", "value": "16", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7021"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0004", "subject_key": "tiering.loop_stmt_limit", "predicate": "loop statement count limit for promotion", "value": "300", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::EnterLoopExpression@L6991-L7021"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} +{"id": "CLM-TIER-0005", "subject_key": "tiering.always_compile_used", "predicate": "AlwaysCompile passed by any caller", "value": "false", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2096"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-001"} +{"id": "CLM-TIER-0006", "subject_key": "tiering.optimized_condition", "predicate": "optimized variant requires", "value": "debugging off and new local scope", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock@L542-L2177"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-003"} +{"id": "CLM-TIER-0007", "subject_key": "tiering.allscope_fallback", "predicate": "optimized variant replaced by unoptimized when", "value": "an allocated variable could be AllScope", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-003"} +{"id": "CLM-TIER-0008", "subject_key": "tiering.forced_jit", "predicate": "forced immediate JIT", "value": "by-ref call/ctor or instance call on non-primitive value type", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightCompiler.cs::LightCompiler@L271-L2052"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-001"} +{"id": "CLM-TIER-0009", "subject_key": "tiering.promotion_async", "predicate": "JIT promotion runs on", "value": "thread pool unless threshold <= 1", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-001"} +{"id": "CLM-TIER-0010", "subject_key": "interpreter.max_parameters", "predicate": "maximum parameters of an interpreted lambda", "value": "16", "label": "FACT-SRC", "evidence": ["src/System.Management.Automation/engine/interpreter/LightLambda.Generated.cs::LightLambda@L27-L28"], "doc": "LANG-TIERING-001", "cf": "CF-CMP-002"} diff --git a/docs-out/data/lessons.json b/docs-out/data/lessons.json index f30f90d..b774b64 100644 --- a/docs-out/data/lessons.json +++ b/docs-out/data/lessons.json @@ -153,6 +153,523 @@ "file": "01-repository/coding-standards.md", "line": 111 }, + { + "id": "LSN-TOK-01", + "verdict": "REPLICATE", + "area": "NLD-LEX", + "text": "Keep a parser-selected `Mode` (Command / Expression / TypeName / Signature) on a pull-style tokenizer; initial mode Command; each switch restored in `finally`.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`.", + "dev": null, + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 301 + }, + { + "id": "LSN-TOK-02", + "verdict": "REPLICATE", + "area": "NLD-LEX", + "text": "Treat `-name` as a parameter in Command mode and as dash-operator-or-parameter in Expression mode; recognize keywords only in Command/Signature mode.", + "evidence": "`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanParameter@L3185-L3338`, `Tokenizer.ScanIdentifier@L4402-L4458`.", + "dev": null, + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 302 + }, + { + "id": "LSN-TOK-03", + "verdict": "REPLICATE", + "area": "NLD-LEX", + "text": "Reproduce the generic-token rule in Command mode: operator characters and non-terminators glue into one word, with the exact terminator set (`ForceStartNewToken`).", + "evidence": "`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanGenericToken@L3382-L3479`, `src/System.Management.Automation/engine/parser/CharTraits.cs::CharExtensions@L230-L413`.", + "dev": null, + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 303 + }, + { + "id": "LSN-TOK-04", + "verdict": "ADAPT", + "area": "NLD-LEX", + "text": "Implement rescan with an explicit snapshot (position + mode) instead of resyncing from a token, but keep the one-token-lookahead contract so a mode switch re-reads the peeked token. Why: same observable behavior, simpler to reason about.", + "evidence": "`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.Resync@L1039-L1078`.", + "dev": null, + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 304 + }, + { + "id": "LSN-TOK-05", + "verdict": "REPLICATE", + "area": "NLD-LEX", + "text": "Number literal grammar: suffix before multiplier, multiplier needs `b`, two's-complement hex/binary, int\u2192long\u2192decimal\u2192double promotion.", + "evidence": "`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.TryGetNumberValue@L3555-L3882`.", + "dev": null, + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 305 + }, + { + "id": "LSN-TOK-06", + "verdict": "ADAPT", + "area": "NLD-LEX", + "text": "Pre-scan `$(...)` inside strings with a tokenizer-aware scan (quotes and comments), not paren counting alone. Why: avoids the early-`)` edge case; observable behavior for valid scripts is identical.", + "evidence": "`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.ScanSubExpression@L2362-L2447`.", + "dev": null, + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 306 + }, + { + "id": "LSN-TOK-07", + "verdict": "AVOID", + "area": "NLD-LEX", + "text": "Leave `#requires` processing and signature lookalike detection out of the tokenizer (move to a later phase or omit); replace with a dedicated pragma pass if the owner wants it.", + "evidence": "`src/System.Management.Automation/engine/parser/tokenizer.cs::Tokenizer.GetScriptRequirements@L1894-L1987`.", + "dev": "DEV-LEX-001", + "doc": "LANG-TOKENIZER-001", + "file": "02-language/02.1-tokenizer.md", + "line": 307 + }, + { + "id": "LSN-PARSER-01", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Hand-written recursive descent with exactly one token of lookahead; the parser, not the tokenizer, owns Command/Expression mode and restores it after each rule.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SetTokenizerMode@L499-L535`.", + "dev": null, + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 420 + }, + { + "id": "LSN-PARSER-02", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "At a statement start try an expression in Expression mode first (assignment target or first pipeline element), then fall back to a command.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.PipelineChainRule@L5773-L5988`.", + "dev": null, + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 421 + }, + { + "id": "LSN-PARSER-03", + "verdict": "REPLICATE", + "area": "NLD-OP", + "text": "Parse binary operators with an operand stack and operator stack driven by a per-operator precedence level, all left-associative; comma binds tighter than every binary operator; ternary and assignment sit above.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.BinaryExpressionRule@L6798-L6950`.", + "dev": null, + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 422 + }, + { + "id": "LSN-PARSER-04", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Do not require a terminator after block statements; skip optional newlines and semicolons between statements, and stop pipelines at newline, `;`, `)`, `}`, `&&`, `||`, `&`.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.StatementListRule@L1897-L1944`.", + "dev": null, + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 423 + }, + { + "id": "LSN-PARSER-05", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Collect errors with de-duplication, return error AST nodes and recover by pretending closers or skipping to a sync token; mark errors at end-of-script as incomplete input so the REPL can continue the line.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.SyncOnError@L366-L432`, `Parser.ReportIncompleteInput@L8111-L8141`.", + "dev": null, + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 424 + }, + { + "id": "LSN-PARSER-06", + "verdict": "ADAPT", + "area": "NLD-GRM", + "text": "Keep stack-depth protection but replace the retry-on-new-thread with a fixed nesting limit that reports `ScriptTooComplicated`. Why: same observable error, deterministic and testable.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ParseTask@L176-L227`.", + "dev": null, + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 425 + }, + { + "id": "LSN-PARSER-07", + "verdict": "AVOID", + "area": "NLD-GRM", + "text": "Do not perform filesystem access or runspace creation while parsing (`using assembly` resolution, `configuration`); resolve such directives in a later phase.", + "evidence": "`src/System.Management.Automation/engine/parser/Parser.cs::Parser.ResolveUsingAssembly@L5088-L5168`.", + "dev": "DEV-GRM-010", + "doc": "LANG-PARSER-001", + "file": "02-language/02.2-parser.md", + "line": 426 + }, + { + "id": "LSN-AST-01", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "One class per construct, get-only properties, constructor invariants that throw on invalid combinations, and parent links set exactly once.", + "evidence": "`src/System.Management.Automation/engine/parser/ast.cs::Ast@L91-L420`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 187 + }, + { + "id": "LSN-AST-02", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Provide a pre-order visitor with `Continue`/`SkipChildren`/`StopVisit` and a post-visit hook that fires on every exit path.", + "evidence": "`src/System.Management.Automation/engine/parser/PreOrderVisitor.cs::AstVisitor@L35-L215`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 188 + }, + { + "id": "LSN-AST-03", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Keep error nodes (`ErrorStatementAst`, `ErrorExpressionAst`) that hold partially parsed children so tooling works on broken scripts.", + "evidence": "`src/System.Management.Automation/engine/parser/ast.cs::ErrorStatementAst@L456-L648`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 189 + }, + { + "id": "LSN-AST-04", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Treat hashtable values as statements and keep `PipelineChainAst` left-nested with a plain `PipelineAst` on the right, as the parser produces them.", + "evidence": "`src/System.Management.Automation/engine/parser/ast.cs::HashtableAst@L10128-L10218`, `PipelineChainAst@L5549-L5650`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 190 + }, + { + "id": "LSN-AST-05", + "verdict": "ADAPT", + "area": "NLD-GRM", + "text": "Use one visitor family with a default `Visit` for new nodes instead of two families gated by \"v2\" interfaces. Why: the v1/v2 split exists for source compatibility with external visitors, which a new shell does not have.", + "evidence": "`src/System.Management.Automation/engine/parser/AstVisitor.cs::ICustomAstVisitor2@L207-L235`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 191 + }, + { + "id": "LSN-AST-06", + "verdict": "ADAPT", + "area": "NLD-GRM", + "text": "Make type names (`ITypeName`) plain syntax objects resolved lazily by the type system, with a cache owned by the resolver. Why: keeps the AST free of reflection state while keeping the observable `[type]` resolution.", + "evidence": "`src/System.Management.Automation/engine/parser/ast.cs::TypeName@L8425-L8724`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 192 + }, + { + "id": "LSN-AST-07", + "verdict": "AVOID", + "area": "NLD-GRM", + "text": "Do not store compiler caches (`TupleIndex`, `RuntimeUsingIndex`, `TypeDefinitionAst.Type`) or assignment-code-generation methods on AST nodes; keep them in side tables keyed by node.", + "evidence": "`src/System.Management.Automation/engine/parser/ast.cs::VariableExpressionAst@L9404-L9624`.", + "dev": "DEV-AST-020", + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 193 + }, + { + "id": "LSN-AST-08", + "verdict": "ADAPT", + "area": "NLD-SEC", + "text": "Provide an allow-list \"literal data\" evaluator (no commands, no member calls, no binary operators, node and key caps) for manifests and data sections, evaluated without the compiler. Why: same observable safety as `SafeGetValue`, without depending on the compiler for unary/convert nodes.", + "evidence": "`src/System.Management.Automation/engine/parser/SafeValues.cs::IsSafeValueVisitor@L39-L356`.", + "dev": null, + "doc": "LANG-AST-001", + "file": "02-language/02.3-ast.md", + "line": 194 + }, + { + "id": "LSN-SEM-01", + "verdict": "REPLICATE", + "area": "NLD-GRM", + "text": "Keep semantic rules (duplicate parameters, loop-control placement, assignment targets, attribute-argument constants, hashtable duplicate keys, `$using` shape) as a post-parse pass that reports into the parser's error list.", + "evidence": "src/System.Management.Automation/engine/parser/SemanticChecks.cs::SemanticChecks@L21-L1514.", + "dev": null, + "doc": "LANG-SEMANTIC-001", + "file": "02-language/02.4-semantic-analysis.md", + "line": 142 + }, + { + "id": "LSN-SEM-02", + "verdict": "REPLICATE", + "area": "NLD-SEC", + "text": "Validate `data` sections with an allow-list checker and refuse operators flagged as disallowed in restricted mode.", + "evidence": "src/System.Management.Automation/engine/parser/SemanticChecks.cs::RestrictedLanguageChecker@L1807-L2454.", + "dev": null, + "doc": "LANG-SEMANTIC-001", + "file": "02-language/02.4-semantic-analysis.md", + "line": 143 + }, + { + "id": "LSN-SEM-03", + "verdict": "REPLICATE", + "area": "NLD-TYP", + "text": "Resolve type names in a fixed order (script types, accelerators, assemblies, user accelerators, `using namespace` retries) and cache results per (name, state).", + "evidence": "src/System.Management.Automation/engine/parser/TypeResolver.cs::TypeResolver@L31-L506.", + "dev": null, + "doc": "LANG-SEMANTIC-001", + "file": "02-language/02.4-semantic-analysis.md", + "line": 144 + }, + { + "id": "LSN-SEM-04", + "verdict": "ADAPT", + "area": "NLD-SCP", + "text": "Do slot allocation as a separate pass with a definite-assignment analysis, but start with a simple rule (slot only if assigned before every read, never for non-local scopes) and keep a dynamic-lookup fallback. Why: same observable variable behavior with far less code than the flow-graph analysis.", + "evidence": "src/System.Management.Automation/engine/parser/VariableAnalysis.cs::VariableAnalysis@L325-L1847.", + "dev": null, + "doc": "LANG-SEMANTIC-001", + "file": "02-language/02.4-semantic-analysis.md", + "line": 145 + }, + { + "id": "LSN-SEM-05", + "verdict": "ADAPT", + "area": "NLD-GRM", + "text": "Report all class-definition errors in the semantic pass instead of at first compile. Why: same error set, earlier and deterministic.", + "evidence": "src/System.Management.Automation/engine/parser/PSType.cs::TypeDefiner@L18-L1505.", + "dev": null, + "doc": "LANG-SEMANTIC-001", + "file": "02-language/02.4-semantic-analysis.md", + "line": 146 + }, + { + "id": "LSN-SEM-06", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Do not run module lookups (`Get-Module -ListAvailable`) while parsing `using module`; record the directive and resolve it in a later phase.", + "evidence": "src/System.Management.Automation/engine/parser/SymbolResolver.cs::SymbolResolver@L275-L729.", + "dev": "DEV-SEM-030", + "doc": "LANG-SEMANTIC-001", + "file": "02-language/02.4-semantic-analysis.md", + "line": 147 + }, + { + "id": "LSN-ENG-01", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Start with a tree-walking evaluator over the AST that implements the protocols of \u00a76 and [IR-SPEC.md](IR-SPEC.md) \u00a75; add an IR later only if profiling demands it. Why: same observable behavior, far less code than a 6,600-line expression-tree compiler.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler@L816-L6661.", + "dev": null, + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 183 + }, + { + "id": "LSN-ENG-02", + "verdict": "REPLICATE", + "area": "NLD-PIP", + "text": "Implement statement output as writes to the current output pipe with temporary capture pipes for conditions, assignments, `$()` and `@()`.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureAstResults@L2264-L2355.", + "dev": null, + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 184 + }, + { + "id": "LSN-ENG-03", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Keep statement-level error handling with resume-at-next-statement, rethrowing flow-control signals untouched.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268.", + "dev": null, + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 185 + }, + { + "id": "LSN-ENG-04", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Match `try/catch/finally` semantics: typed handlers in order, `$_` bound to an `ErrorRecord`, `finally` runs after `return`, cleanup runs even for a stopped pipeline.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitTryStatement@L5247-L5509.", + "dev": null, + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 186 + }, + { + "id": "LSN-ENG-05", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "`switch` evaluates every clause per item; `foreach` wraps a non-enumerable in a one-element sequence; loop labels match case-insensitively with an empty label matching any loop.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitSwitchStatement@L4410-L4542, src/System.Management.Automation/engine/lang/parserutils.cs::LoopFlowException@L36-L67.", + "dev": null, + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 187 + }, + { + "id": "LSN-ENG-06", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Cache parsed script blocks by (file, text) but share only immutable compiled data between clones. Why: PowerShell clears the whole cache at 1024 entries; use an LRU instead.", + "evidence": "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::ScriptBlock.CacheScriptBlock@L609-L637.", + "dev": null, + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 188 + }, + { + "id": "LSN-ENG-07", + "verdict": "AVOID", + "area": "NLD-SEC", + "text": "Do not run an AMSI scan, suspicious-pattern hashing or admin-block checks as part of compilation in MicroteamShell; add a separate policy hook if needed.", + "evidence": "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.PerformSecurityChecks@L197-L283.", + "dev": "DEV-ENG-040", + "doc": "LANG-ENGINE-001", + "file": "02-language/02.5-compiler-and-engine.md", + "line": 189 + }, + { + "id": "LSN-EQV-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Define behavior once, in the evaluator and the protocols, never per execution strategy; if two strategies are ever added, test them differentially.", + "evidence": "src/System.Management.Automation/engine/interpreter/Interpreter.cs::Interpreter@L31-L109.", + "dev": null, + "doc": "LANG-EQUIV-001", + "file": "02-language/EQUIVALENCE-SPEC.md", + "line": 124 + }, + { + "id": "LSN-EQV-02", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "`finally` must run for every exit (return, break, continue, throw) and flow-control signals must never be swallowed by catch-all handlers.", + "evidence": "src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterTryCatchFinallyInstruction@L305-L447.", + "dev": null, + "doc": "LANG-EQUIV-001", + "file": "02-language/EQUIVALENCE-SPEC.md", + "line": 125 + }, + { + "id": "LSN-EQV-03", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not port the typed arithmetic/comparison instruction families or the DLR-derived interpreter; they are unreachable or defective for PowerShell. Replace with the evaluator's own numeric semantics from CONVERSION-RULES (planned T28).", + "evidence": "src/System.Management.Automation/engine/interpreter/Instruction.cs::Instruction@L27-L70.", + "dev": "DEV-EQV-061", + "doc": "LANG-EQUIV-001", + "file": "02-language/EQUIVALENCE-SPEC.md", + "line": 126 + }, + { + "id": "LSN-IR-01", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "A tree-walker over the AST can replace the IR, provided it implements the protocols in \u00a75 (output pipe, `$?`, error-action resume, positions, flow control). Why: the observable behavior is defined by the protocols, not by expression trees.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileSingleLambda@L2552-L2626.", + "dev": null, + "doc": "LANG-IR-001", + "file": "02-language/IR-SPEC.md", + "line": 182 + }, + { + "id": "LSN-IR-02", + "verdict": "REPLICATE", + "area": "NLD-PIP", + "text": "Statements write to an output pipe; assignments, `++`/`--` and conditions do not; capturing output swaps the current pipe and restores it in a `finally`.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CaptureAstResults@L2264-L2355, src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.VisitCommandExpression@L4268-L4279.", + "dev": null, + "doc": "LANG-IR-001", + "file": "02-language/IR-SPEC.md", + "line": 183 + }, + { + "id": "LSN-IR-03", + "verdict": "REPLICATE", + "area": "NLD-ERR", + "text": "Wrap each statement so that non-flow-control exceptions go to the error-action policy and execution continues with the next statement when the policy says so.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileStatementListWithTraps@L3046-L3268.", + "dev": null, + "doc": "LANG-IR-001", + "file": "02-language/IR-SPEC.md", + "line": 184 + }, + { + "id": "LSN-IR-04", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "Represent break/continue/return as control-flow signals inside the evaluator, and as exceptions only when crossing function or script-block boundaries. Why: PowerShell uses labels in-lambda and exceptions across lambdas; a tree-walker has no lambdas but still crosses script-block calls.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.GenerateBreakOrContinue@L5511-L5551.", + "dev": null, + "doc": "LANG-IR-001", + "file": "02-language/IR-SPEC.md", + "line": 185 + }, + { + "id": "LSN-IR-05", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Keep a per-statement sequence-point index in the call frame so errors and the debugger can report positions.", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::UpdatePositionExpr@L7025-L7081.", + "dev": null, + "doc": "LANG-IR-001", + "file": "02-language/IR-SPEC.md", + "line": 186 + }, + { + "id": "LSN-TIER-01", + "verdict": "REPLICATE", + "area": "NLD-RUN", + "text": "Start with a tree-walking evaluator only; it is a valid first tier (all observable behavior is defined by the IR semantics, not by the JIT). `INFERENCE`: PowerShell's tiers are equivalent by design ([EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md)).", + "evidence": "src/System.Management.Automation/engine/parser/Compiler.cs::Compiler.CompileTree@L2079-L2097.", + "dev": null, + "doc": "LANG-TIERING-001", + "file": "02-language/TIERING-POLICY.md", + "line": 147 + }, + { + "id": "LSN-TIER-02", + "verdict": "ADAPT", + "area": "NLD-RUN", + "text": "If a second tier is ever added, trigger it by call count with a deterministic (synchronous) policy first, and keep results independent of the tier. Why: PowerShell's asynchronous promotion makes tier choice nondeterministic.", + "evidence": "src/System.Management.Automation/engine/interpreter/LightLambda.cs::LightLambda.TryGetCompiled@L197-L246.", + "dev": null, + "doc": "LANG-TIERING-001", + "file": "02-language/TIERING-POLICY.md", + "line": 148 + }, + { + "id": "LSN-TIER-03", + "verdict": "REPLICATE", + "area": "NLD-SCP", + "text": "Keep two variable-storage strategies visible in the design: fast slots when the scope is private and analyzable, dynamic lookup when debugging, dot-sourcing, or `AllScope`/`New-Variable`-style access is possible.", + "evidence": "src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs::CompiledScriptBlockData.Compile@L57-L83.", + "dev": null, + "doc": "LANG-TIERING-001", + "file": "02-language/TIERING-POLICY.md", + "line": 149 + }, + { + "id": "LSN-TIER-04", + "verdict": "AVOID", + "area": "NLD-RUN", + "text": "Do not copy background (thread-pool) compilation of hot loops with an in-place instruction replacement; it needs the interpreter and compiled code to share frames exactly. Replace by an explicit, synchronous optimization step if a loop optimizer is wanted.", + "evidence": "src/System.Management.Automation/engine/interpreter/ControlFlowInstructions.cs::EnterLoopInstruction@L692-L778.", + "dev": "DEV-TIER-060", + "doc": "LANG-TIERING-001", + "file": "02-language/TIERING-POLICY.md", + "line": 150 + }, { "id": "LSN-CIM-01", "verdict": "AVOID", diff --git a/docs-out/data/numbers-extra.json b/docs-out/data/numbers-extra.json index 0637a08..93a0eaf 100644 --- a/docs-out/data/numbers-extra.json +++ b/docs-out/data/numbers-extra.json @@ -1 +1,121 @@ -[] \ No newline at end of file +[ + { + "key": "TOKENIZER_KEYWORD_ROWS", + "cmd": "python3 docs-out/tools/gen_T21.py --count keywords", + "dataset": "tables.json", + "label": "APPROX", + "notes": "rows of Tokenizer.s_keywordText (regex-parsed array literal)" + }, + { + "key": "TOKENIZER_DASH_OPERATOR_ROWS", + "cmd": "python3 docs-out/tools/gen_T21.py --count dash_operators", + "dataset": "tables.json", + "label": "APPROX", + "notes": "rows of Tokenizer._operatorText" + }, + { + "key": "PARSER_METHODS", + "cmd": "python3 docs-out/tools/gen_T22.py --count parser_methods", + "dataset": "parser-methods.json", + "label": "APPROX", + "notes": "methods of Parser.cs per scanner" + }, + { + "key": "BINARY_OPERATORS", + "cmd": "python3 docs-out/tools/gen_T22.py --count binary_operators", + "dataset": "tables.json", + "label": "APPROX", + "notes": "TokenKinds with BinaryOperator flag in s_staticTokenFlags" + }, + { + "key": "PARSERSTRINGS_KEYS", + "cmd": "python3 docs-out/tools/gen_T22.py --count parserstrings_keys", + "dataset": "resources.json", + "label": "APPROX", + "notes": "keys of ParserStrings.resx" + }, + { + "key": "PARSERSTRINGS_KEYS_PARSER_TOKENIZER", + "cmd": "python3 docs-out/tools/gen_T22.py --count parserstrings_keys_parser_tokenizer", + "dataset": "resources.json", + "label": "APPROX", + "notes": "ParserStrings keys with a use site in Parser.cs or tokenizer.cs" + }, + { + "key": "AST_CLASSES_ASTCS", + "cmd": "python3 docs-out/tools/gen_T23.py --count ast_classes", + "dataset": "ast-hierarchy.json", + "label": "APPROX", + "notes": "Ast-derived classes declared in ast.cs" + }, + { + "key": "AST_CLASSES_ABSTRACT", + "cmd": "python3 docs-out/tools/gen_T23.py --count abstract", + "dataset": "ast-hierarchy.json", + "label": "APPROX", + "notes": "abstract classes among them" + }, + { + "key": "ICUSTOMASTVISITOR_METHODS", + "cmd": "python3 docs-out/tools/gen_T23.py --count iv1", + "dataset": "visitors.json", + "label": "APPROX", + "notes": "Visit methods in ICustomAstVisitor" + }, + { + "key": "ICUSTOMASTVISITOR2_METHODS", + "cmd": "python3 docs-out/tools/gen_T23.py --count iv2", + "dataset": "visitors.json", + "label": "APPROX", + "notes": "Visit methods added by ICustomAstVisitor2" + }, + { + "key": "SEMANTIC_CHECK_SITES", + "cmd": "python3 docs-out/tools/gen_T24.py --count sites", + "dataset": "types.json", + "label": "APPROX", + "notes": "nameof(ParserStrings.X) sites in SemanticChecks.cs, SymbolResolver.cs, PSType.cs" + }, + { + "key": "SEMANTIC_CHECK_IDS", + "cmd": "python3 docs-out/tools/gen_T24.py --count ids", + "dataset": "types.json", + "label": "APPROX", + "notes": "distinct ids among them" + }, + { + "key": "COMPILER_VISIT_METHODS", + "cmd": "python3 docs-out/tools/gen_T25.py --count visits", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "Visit* methods of Compiler" + }, + { + "key": "COMPILER_HELPER_METHODS", + "cmd": "python3 docs-out/tools/gen_T25.py --count helpers", + "dataset": "members-tierA.json", + "label": "APPROX", + "notes": "non-Visit methods of Compiler" + }, + { + "key": "INSTRUCTION_CLASSES", + "cmd": "python3 docs-out/tools/gen_T27.py --count instructions", + "dataset": "instructions.json", + "label": "APPROX", + "notes": "classes deriving from Instruction found by scanner" + }, + { + "key": "INSTRUCTION_ABSTRACT", + "cmd": "python3 docs-out/tools/gen_T27.py --count abstract", + "dataset": "instructions.json", + "label": "APPROX", + "notes": "abstract ones" + }, + { + "key": "INSTRUCTION_FILES", + "cmd": "python3 docs-out/tools/gen_T27.py --count files", + "dataset": "instructions.json", + "label": "APPROX", + "notes": "files holding them" + } +] \ No newline at end of file diff --git a/docs-out/data/numbers.json b/docs-out/data/numbers.json index 8cf5f4d..229337d 100644 --- a/docs-out/data/numbers.json +++ b/docs-out/data/numbers.json @@ -367,5 +367,158 @@ "evidence": "E1", "label": "APPROX", "notes": "methods in Parser.cs + tokenizer.cs" + }, + { + "key": "TOKENIZER_KEYWORD_ROWS", + "value": 51, + "dataset": "tables.json", + "command": "python3 docs-out/tools/gen_T21.py --count keywords", + "evidence": "E1", + "label": "APPROX", + "notes": "rows of Tokenizer.s_keywordText (regex-parsed array literal)" + }, + { + "key": "TOKENIZER_DASH_OPERATOR_ROWS", + "value": 63, + "dataset": "tables.json", + "command": "python3 docs-out/tools/gen_T21.py --count dash_operators", + "evidence": "E1", + "label": "APPROX", + "notes": "rows of Tokenizer._operatorText" + }, + { + "key": "PARSER_METHODS", + "value": 157, + "dataset": "parser-methods.json", + "command": "python3 docs-out/tools/gen_T22.py --count parser_methods", + "evidence": "E1", + "label": "APPROX", + "notes": "methods of Parser.cs per scanner" + }, + { + "key": "BINARY_OPERATORS", + "value": 52, + "dataset": "tables.json", + "command": "python3 docs-out/tools/gen_T22.py --count binary_operators", + "evidence": "E1", + "label": "APPROX", + "notes": "TokenKinds with BinaryOperator flag in s_staticTokenFlags" + }, + { + "key": "PARSERSTRINGS_KEYS", + "value": 412, + "dataset": "resources.json", + "command": "python3 docs-out/tools/gen_T22.py --count parserstrings_keys", + "evidence": "E1", + "label": "APPROX", + "notes": "keys of ParserStrings.resx" + }, + { + "key": "PARSERSTRINGS_KEYS_PARSER_TOKENIZER", + "value": 157, + "dataset": "resources.json", + "command": "python3 docs-out/tools/gen_T22.py --count parserstrings_keys_parser_tokenizer", + "evidence": "E1", + "label": "APPROX", + "notes": "ParserStrings keys with a use site in Parser.cs or tokenizer.cs" + }, + { + "key": "AST_CLASSES_ASTCS", + "value": 76, + "dataset": "ast-hierarchy.json", + "command": "python3 docs-out/tools/gen_T23.py --count ast_classes", + "evidence": "E1", + "label": "APPROX", + "notes": "Ast-derived classes declared in ast.cs" + }, + { + "key": "AST_CLASSES_ABSTRACT", + "value": 11, + "dataset": "ast-hierarchy.json", + "command": "python3 docs-out/tools/gen_T23.py --count abstract", + "evidence": "E1", + "label": "APPROX", + "notes": "abstract classes among them" + }, + { + "key": "ICUSTOMASTVISITOR_METHODS", + "value": 54, + "dataset": "visitors.json", + "command": "python3 docs-out/tools/gen_T23.py --count iv1", + "evidence": "E1", + "label": "APPROX", + "notes": "Visit methods in ICustomAstVisitor" + }, + { + "key": "ICUSTOMASTVISITOR2_METHODS", + "value": 9, + "dataset": "visitors.json", + "command": "python3 docs-out/tools/gen_T23.py --count iv2", + "evidence": "E1", + "label": "APPROX", + "notes": "Visit methods added by ICustomAstVisitor2" + }, + { + "key": "SEMANTIC_CHECK_SITES", + "value": 133, + "dataset": "types.json", + "command": "python3 docs-out/tools/gen_T24.py --count sites", + "evidence": "E1", + "label": "APPROX", + "notes": "nameof(ParserStrings.X) sites in SemanticChecks.cs, SymbolResolver.cs, PSType.cs" + }, + { + "key": "SEMANTIC_CHECK_IDS", + "value": 104, + "dataset": "types.json", + "command": "python3 docs-out/tools/gen_T24.py --count ids", + "evidence": "E1", + "label": "APPROX", + "notes": "distinct ids among them" + }, + { + "key": "COMPILER_VISIT_METHODS", + "value": 63, + "dataset": "members-tierA.json", + "command": "python3 docs-out/tools/gen_T25.py --count visits", + "evidence": "E1", + "label": "APPROX", + "notes": "Visit* methods of Compiler" + }, + { + "key": "COMPILER_HELPER_METHODS", + "value": 99, + "dataset": "members-tierA.json", + "command": "python3 docs-out/tools/gen_T25.py --count helpers", + "evidence": "E1", + "label": "APPROX", + "notes": "non-Visit methods of Compiler" + }, + { + "key": "INSTRUCTION_CLASSES", + "value": 183, + "dataset": "instructions.json", + "command": "python3 docs-out/tools/gen_T27.py --count instructions", + "evidence": "E1", + "label": "APPROX", + "notes": "classes deriving from Instruction found by scanner" + }, + { + "key": "INSTRUCTION_ABSTRACT", + "value": 17, + "dataset": "instructions.json", + "command": "python3 docs-out/tools/gen_T27.py --count abstract", + "evidence": "E1", + "label": "APPROX", + "notes": "abstract ones" + }, + { + "key": "INSTRUCTION_FILES", + "value": 22, + "dataset": "instructions.json", + "command": "python3 docs-out/tools/gen_T27.py --count files", + "evidence": "E1", + "label": "APPROX", + "notes": "files holding them" } ] \ No newline at end of file diff --git a/docs-out/data/subject-keys.json b/docs-out/data/subject-keys.json index 9e26dfe..568007b 100644 --- a/docs-out/data/subject-keys.json +++ b/docs-out/data/subject-keys.json @@ -1 +1,79 @@ -{} \ No newline at end of file +{ + "ast.assignment.rhs_normalization": "", + "ast.constant.evaluation": "", + "ast.find.nested_skip": "", + "ast.find.root_tested": "", + "ast.hashtable.value_type": "", + "ast.parent.set_once": "", + "ast.pipelinechain.shape": "", + "ast.safevalue.binary_expression": "", + "ast.safevalue.max_hashtable_keys": "", + "ast.safevalue.max_visits": "", + "ast.ternary.visitor_gate": "", + "ast.visit.pre_order_post_hook": "", + "ast.visit.v2_nodes": "", + "compiler.foreach.enumeration": "", + "compiler.log_segment": "", + "compiler.loop.nonlocal_break": "", + "compiler.return_semantics": "", + "compiler.scriptblock.cache_cap": "", + "compiler.single_expression.tier": "", + "compiler.statement_dispatch": "", + "compiler.switch.clauses": "", + "compiler.visit_method_count": "", + "interpreter.max_parameters": "", + "parser.array_type.max_dimensions": "", + "parser.assignment.rhs": "", + "parser.binary_operators.associativity": "", + "parser.chain_operators.associativity": "", + "parser.class.constrained_language": "", + "parser.comma.vs_binary": "", + "parser.errors.dedup_key": "", + "parser.errors.max_count": "", + "parser.hashtable.value": "", + "parser.lookahead.tokens": "", + "parser.member_access.whitespace": "", + "parser.mode_analysis.post_parse_checks": "", + "parser.stack.retry": "", + "parser.statement.start": "", + "parser.statement.terminator_after_block": "", + "parser.switch.flags_matching": "", + "parser.using_assembly.resolution_time": "", + "restricted.default_variables": "", + "semantic.array_type.max_depth": "", + "semantic.order": "", + "semantic.unresolved_type_error": "", + "tiering.allscope_fallback": "", + "tiering.always_compile_used": "", + "tiering.call_threshold": "", + "tiering.forced_jit": "", + "tiering.loop_iterations": "", + "tiering.loop_stmt_limit": "", + "tiering.optimized_condition": "", + "tiering.promotion_async": "", + "tiering.stmt_limit": "", + "tokenizer.chain_operators.mode_dependent": "", + "tokenizer.comments.block_nest": "", + "tokenizer.herestring.footer": "", + "tokenizer.line_endings": "", + "tokenizer.lookahead.depth": "", + "tokenizer.mode.dash_operators_recognized_in": "", + "tokenizer.mode.initial": "", + "tokenizer.mode.keywords_recognized_in": "", + "tokenizer.mode.switch_rescans_lookahead": "", + "tokenizer.numeric.default_promotion": "", + "tokenizer.numeric.multiplier_base": "", + "tokenizer.numeric.multiplier_requires_b": "", + "tokenizer.numeric.signed_requires_flag": "", + "tokenizer.numeric.type_suffixes": "", + "tokenizer.questionmark.expression_only": "", + "tokenizer.redirection.stream_digits": "", + "tokenizer.requires.recognition": "", + "tokenizer.string.single_quote_escape": "", + "tokenizer.subexpr.prescan": "", + "typedefiner.run_time": "", + "typeresolver.order": "", + "variableanalysis.force_dynamic": "", + "variableanalysis.pessimizing_cmdlets": "", + "variableanalysis.unanalyzed": "" +} \ No newline at end of file diff --git a/docs-out/tools/__pycache__/gen_T21.cpython-313.pyc b/docs-out/tools/__pycache__/gen_T21.cpython-313.pyc new file mode 100644 index 0000000..2728e9f Binary files /dev/null and b/docs-out/tools/__pycache__/gen_T21.cpython-313.pyc differ diff --git a/docs-out/tools/annotations/T25-compilation-map.json b/docs-out/tools/annotations/T25-compilation-map.json new file mode 100644 index 0000000..983c697 --- /dev/null +++ b/docs-out/tools/annotations/T25-compilation-map.json @@ -0,0 +1,68 @@ +{ +"_note": "Hand-written annotations for COMPILATION-MAP.md (task T25). Keys are Compiler.Visit* method names / helper names. Each text was written from reading Compiler.cs (FACT-SRC); see .work/notes/src_System.Management.Automation_engine_parser_Compiler.cs.md.", +"visits": { +"VisitErrorStatement": ["not compiled", "returns null; error nodes never reach the compiler because a script with parse errors cannot be turned into a script block"], +"VisitErrorExpression": ["constant", "returns Constant(1); placeholder"], +"VisitScriptBlock": ["five lambdas", "builds one Expression> per named block (DynamicParam, Begin, Process, End, Clean); function name suffixes `` etc.; param block is not compiled (defaults come from parameter metadata); the first compiled block also defines classes/usings when the script block is the root"], +"VisitTypeConstraint": ["not visited", "assert false; type constraints are handled by their parents"], +"VisitAttribute": ["not visited", "assert false; attributes are turned into attribute objects by GetAttribute at metadata time"], +"VisitNamedAttributeArgument": ["not visited", "assert false"], +"VisitParameter": ["not visited", "assert false; parameters become RuntimeDefinedParameter metadata (GetParameterMetaData)"], +"VisitParamBlock": ["not visited", "assert false"], +"VisitNamedBlock": ["not visited", "assert false; CompileNamedBlock is called from VisitScriptBlock"], +"VisitStatementBlock": ["statement list", "CompileStatementListWithTraps -> Block (Empty if no statements); each statement wrapped for error-action handling"], +"VisitTypeDefinition": ["empty", "Empty; types are defined by GenerateTypesAndUsings for the root block"], +"VisitPropertyMember": ["not compiled", "returns null"], +"VisitFunctionMember": ["not compiled", "returns null; member bodies are compiled as separate script blocks with member-function mode"], +"VisitBaseCtorInvokeMemberExpression": ["call", "InvokeBaseCtorMethod: dynamic site with PSInvokeBaseCtorBinder"], +"VisitUsingStatement": ["empty", "Empty; usings are loaded by GenerateLoadUsings in the root prolog"], +"VisitConfigurationDefinition": ["synthesized pipeline", "compiles the AST built by GenerateSetItemPipelineAst (set-item function:); DSC, out of scope"], +"VisitDynamicKeywordStatement": ["synthesized pipeline", "meta statements -> Empty; otherwise compiles the AST built by GenerateCommandCallPipelineAst; DSC, out of scope"], +"VisitFunctionDefinition": ["runtime call", "FunctionOps.DefineFunction(context, ast, ScriptBlockExpressionWrapper); the function body is not compiled with the parent, only when the wrapper's script block first runs"], +"VisitIfStatement": ["conditional", "nested Expression.IfThenElse from last clause to first; condition = CaptureStatementResults(cond, Condition) converted to bool; position update before each condition"], +"VisitTrap": ["not visited", "assert false; traps are compiled by CompileTrap from statement lists"], +"VisitAssignmentStatement": ["assignment", "CompileAssignment: destructuring (array LHS) uses PSArrayAssignmentRHSBinder; otherwise ReduceAssignment through IAssignableValue (SetValue / dynamic binary op for +=, -=, ...)"], +"VisitPipelineChain": ["dispatch loop", "&& / || over pipelines: statement-index dispatch with try/catch (like statement lists), `if ($?)` / `if (-not $?)` between elements, background chains call PipelineOps.InvokePipelineInBackground; no trap handlers are pushed"], +"VisitPipeline": ["runtime call", "single expression -> compiled expression (with redirections via GetRedirectedExpression); otherwise PipelineOps.InvokePipeline(input, ignoreInput, CommandParameterInternal[][] per command, command asts, redirections, funcContext); background -> InvokePipelineInBackground; position update unless parent is assignment/paren"], +"VisitMergingRedirection": ["runtime object", "returns a MergingRedirection value (not an Expression)"], +"VisitFileRedirection": ["new object", "Expression.New(FileRedirection(stream, append, fileName)); non-constant file names converted with PSToStringBinder"], +"VisitCommand": ["array of arguments", "NewArrayInit(CommandParameterInternal[]): first element is the command name argument, then parameters/arguments; numeric literals keep their source text (WrappedNumber); strict-mode check for `f (args)` call syntax"], +"VisitCommandExpression": ["expression + output", "compile expression; unless pre/post increment or void, write value to funcContext._outputPipe (CallAddPipe)"], +"VisitCommandParameter": ["argument object", "CommandParameterInternal.CreateParameter / CreateParameterWithArgument"], +"VisitSwitchStatement": ["loop", "saves/restores `$_`; -file reads lines with StreamReader; otherwise GenerateIteratorStatement with `$switch`; per item every clause is tested (constant script block, regex, wildcard, exact string, or PSSwitchClauseEvalBinder) - all matching clauses run; default clause when none matched"], +"VisitDataStatement": ["try/finally", "sets context.LanguageMode to RestrictedLanguage around the body, runtime checks for allowed commands, assigns result to the data variable"], +"VisitForEachStatement": ["loop", "GenerateIteratorStatement with `$foreach`; enumerator via PSEnumerableBinder; body assigns the loop variable then runs the body"], +"VisitDoWhileStatement": ["loop", "GenerateDoLoop: body, then test; jump back while true"], +"VisitDoUntilStatement": ["loop", "GenerateDoLoop with the test negated"], +"VisitForStatement": ["loop", "initializer; GenerateWhileLoop with condition and iterator as the `continue` action"], +"VisitWhileStatement": ["loop", "GenerateWhileLoop"], +"VisitCatchClause": ["not visited", "assert false; catch bodies are compiled by VisitTryStatement"], +"VisitTryStatement": ["try/catch/finally", "ExpressionType.Try; catch-all-only form catches RuntimeException (PipelineStoppedException rethrown), typed catches use ExceptionHandlingOps.FindMatchingHandler + switch; `$_` set to the ErrorRecord; finally wraps SuspendStoppingPipeline/RestoreStoppingPipeline"], +"VisitBreakStatement": ["goto or throw", "GenerateBreakOrContinue: Expression.Break to the matching compiled loop label, else throw BreakException(label)"], +"VisitContinueStatement": ["goto or throw", "same, with ContinueException"], +"VisitReturnStatement": ["return label", "writes the value to the output pipe (or return pipe in class methods) then Expression.Return to the block's return label; inside a trap lambda throws ReturnException"], +"VisitExitStatement": ["throw", "throws the exception from PipelineOps.GetExitException(code)"], +"VisitThrowStatement": ["throw", "ExceptionHandlingOps.ConvertToException(value, extent, isRethrow); `throw` with no value rethrows the exception being handled"], +"VisitTernaryExpression": ["conditional", "Expression.Condition(cond as bool, ifTrue as object, ifFalse as object)"], +"VisitBinaryExpression": ["operator call or dynamic site", "constant-folded when constant; -and/-or -> AndAlso/OrElse over bool; double,double + - * / native; other arithmetic and comparisons via DynamicExpression with PSBinaryOperationBinder; -like/-match/-replace/-split/-join/-contains/-in/-is/-as/-f/.. -> ParserOps/StringOps/TypeOps helper calls; ?? -> Coalesce"], +"VisitUnaryExpression": ["operator call or dynamic site", "constant-folded when constant; -not/! PSUnaryOperationBinder; unary +/- as 0+x / 0-x; -bnot OnesComplement; ++/-- CompileIncrementOrDecrement; unary -join/-split helper calls"], +"VisitConvertExpression": ["conversion", "constant-folded; [ordered]@{} -> OrderedDictionary; [pscustomobject]@{} -> converter; [ref]$x -> VariableOps.GetVariableAsRef; otherwise ConvertValue: Expression.Convert or PSConvertBinder, PSDynamicConvertBinder when the type is unresolved at compile time"], +"VisitConstantExpression": ["constant", "Expression.Constant(value)"], +"VisitStringConstantExpression": ["constant", "Expression.Constant(value)"], +"VisitSubExpression": ["captured block", "no statements -> $null; else CaptureAstResults (temporary output pipe collecting results)"], +"VisitUsingExpression": ["runtime call", "VariableOps.GetUsingValue(locals, key, RuntimeUsingIndex, context)"], +"VisitVariableExpression": ["slot or runtime lookup", "$null/$true/$false constants; automatic variables via tuple slot or VariableOps.GetAutomaticVariableValue; TupleIndex < 0 -> VariableOps.GetVariableValue (by name); else tuple field access"], +"VisitTypeExpression": ["constant or runtime call", "reflection Type constant if resolvable at compile time, else TypeOps.ResolveTypeName at run time"], +"VisitMemberExpression": ["field/property or dynamic site", "static [T]::Member with a unique public static field/property -> direct access; else PSGetMemberBinder / PSGetDynamicMemberBinder; `?.` wrapped in a null check"], +"VisitInvokeMemberExpression": ["dynamic site", "InvokeMember: PSInvokeMemberBinder, PSCreateInstanceBinder for `[T]::new`, PSInvokeDynamicMemberBinder for non-constant names; argument/target type constraints passed to the binder"], +"VisitArrayExpression": ["captured block", "@( ) : empty -> object[0]; single pure expression -> that expression wrapped to array; else CaptureAstResults(Enumerable)"], +"VisitArrayLiteral": ["array init", "Expression.NewArrayInit(object, elements cast to object)"], +"VisitHashtable": ["block", "new Hashtable(count, OrdinalIgnoreCase) then HashtableOps.AddKeyValuePair per pair; each value statement captured with CaptureStatementResults"], +"VisitScriptBlockExpression": ["runtime call", "ScriptBlockExpressionWrapper.GetScriptBlock(context, false); the block body is compiled lazily on first run"], +"VisitParenExpression": ["captured pipeline", "assignment -> CompileAssignment; otherwise CaptureStatementResults of the pipeline"], +"VisitExpandableStringExpression": ["format call", "StringOps.FormatOperator(format string, string[] of nested expressions converted with PSToStringBinder)"], +"VisitIndexExpression": ["dynamic site", "PSGetIndexBinder with 1 or n index arguments; `?[]` wrapped in a null check"], +"VisitAttributedExpression": ["delegate", "child.Accept(this); attributes only matter on assignment targets"], +"VisitBlockStatement": ["not compiled", "returns null (workflow remnants)"] +} +} diff --git a/docs-out/tools/expand_refs.py b/docs-out/tools/expand_refs.py new file mode 100644 index 0000000..8f7b96d --- /dev/null +++ b/docs-out/tools/expand_refs.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""expand_refs.py - expand short reference tokens in a draft into validated evidence references. + +Purpose : authors write [[engine/parser/Compiler.cs::Compiler.VisitIfStatement]] or [[File.cs::Type]] + (file name alone means engine/parser/); this tool replaces each token with + `src/System.Management.Automation//File.cs::Type.Member@Lstart-Lend` using the line ranges in + data/types.json and data/members-tierA.json. A token that cannot be resolved is left unchanged and + reported (exit code 1), so no reference is ever invented. +Inputs : a draft markdown file (argv[1]), data/types.json, data/members-tierA.json +Outputs : writes the expanded text to argv[2] +Run : python3 docs-out/tools/expand_refs.py +Label : ranges are APPROX (scanner); `validate-refs` re-checks existence and plausibility. +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +DATA = os.path.join(ROOT, "docs-out", "data") +BASE = "src/System.Management.Automation/" +TOKEN = re.compile(r"\[\[([\w/\.\-]+\.cs)::([\w\.]+)\]\]") + + +def main(): + types = json.load(open(os.path.join(DATA, "types.json"))) + members = {(m["file"], m["full_name"]): m["members"] for m in json.load(open(os.path.join(DATA, "members-tierA.json")))} + by = {} + for t in types: + by.setdefault(t["file"], []).append(t) + text = open(sys.argv[1], encoding="utf-8").read() + bad = [] + + def sub(m): + f, ref = m.group(1), m.group(2) + if "/" not in f: + f = "engine/parser/" + f + elif f.split("/")[0] in ("interpreter", "runtime", "lang", "parser"): + f = "engine/" + f + path = BASE + f + tn, _, mn = ref.partition(".") + cands = [t for t in by.get(path, []) if t["full_name"].split("Language.")[-1].split("Automation.")[-1] == tn or t["full_name"].split(".")[-1] == tn] + if mn and not cands: + # nested type form Outer.Inner as type + cands = [t for t in by.get(path, []) if t["full_name"].endswith("." + ref)] + if cands: + mn = "" + if not cands: + bad.append(m.group(0)) + return m.group(0) + t = sorted(cands, key=lambda x: x["end"] - x["start"])[0] + if not mn: + return f"{path}::{ref}@L{t['start']}-L{t['end']}" + mm = [x for x in members.get((path, t["full_name"]), []) if x["name"] == mn and x.get("start")] + if not mm: + bad.append(m.group(0)) + return m.group(0) + x = mm[0] + return f"{path}::{ref}@L{x['start']}-L{x['end']}" + + out = TOKEN.sub(sub, text) + open(sys.argv[2], "w", encoding="utf-8").write(out) + for b in bad: + print("UNRESOLVED", b) + print("expanded", len(TOKEN.findall(text)) - len(bad), "unresolved", len(bad)) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T21.py b/docs-out/tools/gen_T21.py new file mode 100644 index 0000000..c4a0b33 --- /dev/null +++ b/docs-out/tools/gen_T21.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""gen_T21.py - render 02-language/TOKEN-CATALOG.md from data/*.json (task T21, rule 9). + +Purpose : every catalog in the tokenizer documents is GENERATED, never hand-typed. + Renders: TokenKind table (value, name, default text, static flags), + TokenFlags table, keyword table, dash-operator table, CharTraits table, + NumberSuffixFlags table, TokenizerMode table. +Inputs : data/enums.json, data/tables.json (verbatim source text of static arrays), data/types.json +Outputs : docs-out/02-language/TOKEN-CATALOG.md (front matter + body) +Run : python3 docs-out/tools/gen_T21.py (from the repo root) + python3 docs-out/tools/gen_T21.py --count keywords|dash_operators (numbers proposals) +Label : every row is APPROX (regex extraction of array literals; cross-checked by length == last + TokenKind value + 1, asserted below). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +TOK = "src/System.Management.Automation/engine/parser/tokenizer.cs" +TKN = "src/System.Management.Automation/engine/parser/token.cs" +CHT = "src/System.Management.Automation/engine/parser/CharTraits.cs" + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def strip_comments(t): + # defect workaround: data/tables.json "text" of s_tokenText/s_staticTokenFlags runs past the + # closing "};" of the array (extractor end-line is too late) -> cut at the first "};". + k = t.find("};") + if k != -1: + t = t[:k + 1] + t = re.sub(r"^\s*#(end)?region[^\n]*$", "", t, flags=re.M) + t = re.sub(r"/\*.*?\*/", "", t, flags=re.S) + t = re.sub(r"//[^\n]*", "", t) + return t + + +def split_top(t): + """split an array literal body on top-level commas (strings may contain commas).""" + t = t.strip() + if t.startswith("{"): + t = t[1:] + if t.endswith("}"): + t = t[:-1] + out, cur, q = [], [], False + i = 0 + while i < len(t): + c = t[i] + if q: + cur.append(c) + if c == "\\": + i += 1 + cur.append(t[i]) + elif c == '"': + q = False + else: + if c == '"': + q = True + cur.append(c) + elif c == ",": + out.append("".join(cur).strip()) + cur = [] + else: + cur.append(c) + i += 1 + last = "".join(cur).strip() + if last: + out.append(last) + return [x for x in out] + + +def unq(s): + s = s.strip() + if s.startswith("@"): + s = s[1:] + if s.startswith('"') and s.endswith('"'): + s = s[1:-1] + if s == "string.Empty": + return "" + return s.replace("\\\\", "\\").replace('\\"', '"') + + +def esc(s): + return s.replace("|", "\\|") + + +def main(): + if len(sys.argv) > 2 and sys.argv[1] == "--count": + tb = J("tables.json") + key = {"keywords": f"{TOK}::s_keywordText", "dash_operators": f"{TOK}::_operatorText"}[sys.argv[2]] + print(len(split_top(strip_comments(tb[key]["text"])))) + return 0 + enums = {e["full_name"].split(".")[-1]: e for e in J("enums.json") + if e["file"].endswith(("parser/token.cs", "parser/tokenizer.cs", "parser/CharTraits.cs"))} + tables = J("tables.json") + kind = enums["TokenKind"] + flags = enums["TokenFlags"] + mem = kind["members"] + nvals = int(mem[-1]["value"]) + 1 + + texts = [unq(x) for x in split_top(strip_comments(tables[f"{TKN}::s_tokenText"]["text"]))] + sflags = [re.sub(r"\s+", " ", x) for x in split_top(strip_comments(tables[f"{TKN}::s_staticTokenFlags"]["text"]))] + assert len(texts) == nvals, (len(texts), nvals) + assert len(sflags) == nvals, (len(sflags), nvals) + byval = {int(m["value"]): m for m in mem} + + kw_text = [unq(x) for x in split_top(strip_comments(tables[f"{TOK}::s_keywordText"]["text"]))] + kw_kind = [x.replace("TokenKind.", "") for x in split_top(strip_comments(tables[f"{TOK}::s_keywordTokenKind"]["text"]))] + op_text = [unq(x) for x in split_top(strip_comments(tables[f"{TOK}::_operatorText"]["text"]))] + op_kind = [x.replace("TokenKind.", "") for x in split_top(strip_comments(tables[f"{TOK}::s_operatorTokenKind"]["text"]))] + assert len(kw_text) == len(kw_kind) and len(op_text) == len(op_kind) + + # keyword kind -> flags + flag_of = {byval[v]["name"]: sflags[v] for v in byval} + + L = [] + w = L.append + w("""--- +id: LANG-TOKCAT-001 +title: Token catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/token.cs, src/System.Management.Automation/engine/parser/tokenizer.cs, src/System.Management.Automation/engine/parser/CharTraits.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/enums.json, data/tables.json] +--- + +# Token catalog + +> STATIC-ONLY — behavior not run-verified. Every table below is `GENERATED` by `docs-out/tools/gen_T21.py` from `data/enums.json` and `data/tables.json` (`APPROX`: the arrays are parsed from verbatim source text; lengths are asserted equal to the `TokenKind` value range). Narrative lives in [02.1-tokenizer.md](02.1-tokenizer.md). Label for every row: `FACT-SRC` (read in source) + `APPROX` (extraction). + +## 1. Purpose and scope [SPEC] + +Complete list of token kinds, token flags, keywords, dash operators, character classes, number-suffix flags and tokenizer modes of the PowerShell tokenizer (v7.6.9). Counts: `TokenKind` has 157 named members spread over the value range 0..%d; `TokenFlags` has 28 members. + +## 2. Source map + +| path | type | role | +|---|---|---| +| `%s::TokenKind@L%d-L%d` | enum | every token kind | +| `%s::TokenFlags@L%d-L%d` | `[Flags]` enum | token traits | +| `%s::TokenTraits.s_staticTokenFlags@L%d-L%d` | table | static flags per kind | +| `%s::TokenTraits.s_tokenText@L%d-L%d` | table | default text per kind | +| `%s::Tokenizer.s_keywordText@L%d-L%d` | table | keyword spelling | +| `%s::Tokenizer._operatorText@L%d-L%d` | table | dash-operator spelling | +| `%s::CharTraits@L%d-L%d` | `[Flags]` enum | per-character class bits | +""" % (nvals - 1, + TKN, kind["start"], kind["end"], + TKN, flags["start"], flags["end"], + TKN, tables[f"{TKN}::s_staticTokenFlags"]["start"], tables[f"{TKN}::s_staticTokenFlags"]["end"], + TKN, tables[f"{TKN}::s_tokenText"]["start"], tables[f"{TKN}::s_tokenText"]["end"], + TOK, tables[f"{TOK}::s_keywordText"]["start"], tables[f"{TOK}::s_keywordText"]["end"], + TOK, tables[f"{TOK}::_operatorText"]["start"], tables[f"{TOK}::_operatorText"]["end"], + CHT, enums["CharTraits"]["start"], enums["CharTraits"]["end"])) + + w("\n## 3. TokenKind (every member) [SPEC]\n") + w("Value gaps are reserved slots (placeholders `Reserved slot N` in the static tables); they are listed so the value range is complete. `Text` is the entry of `s_tokenText`; `Static flags` is the entry of `s_staticTokenFlags`.\n") + w("| Value | Name | Source line | Text | Static flags |") + w("|---:|---|---:|---|---|") + for v in range(nvals): + m = byval.get(v) + if m is None: + w(f"| {v} | *(unnamed: reserved slot)* | — | `{esc(texts[v])}` | {esc(sflags[v].replace('TokenFlags.', ''))} |") + else: + w(f"| {v} | `{m['name']}` | L{m['line']} | `{esc(texts[v])}` | {esc(sflags[v].replace('TokenFlags.', ''))} |") + + w("\n## 4. TokenFlags (every member) [SPEC]\n") + w("| Name | Value | Source line |") + w("|---|---|---:|") + for m in flags["members"]: + w(f"| `{m['name']}` | `{m['value']}` | L{m['line']} |") + + w("\n## 5. Keywords (`Tokenizer.s_keywordText` / `s_keywordTokenKind`) [SPEC]\n") + w("Parallel arrays; recognition is case-insensitive (`s_keywordTable` is built with `OrdinalIgnoreCase`, see [02.1-tokenizer.md](02.1-tokenizer.md) §6). `inlinescript` is only a keyword inside a workflow context.\n") + w("| # | Spelling | TokenKind | Static flags of the kind |") + w("|---:|---|---|---|") + for i, (t, k) in enumerate(zip(kw_text, kw_kind), 1): + w(f"| {i} | `{esc(t)}` | `{k}` | {esc(flag_of.get(k, '?').replace('TokenFlags.', ''))} |") + w(f"\nKeyword rows: **{len(kw_text)}** (`GENERATED`). `TokenKind.DynamicKeyword` has no row; dynamic (DSC) keywords come from `DynamicKeyword.DynamicKeywords`.") + + w("\n## 6. Dash operators (`Tokenizer._operatorText` / `s_operatorTokenKind`) [SPEC]\n") + w("Spelling is stored **without** the leading dash. Recognition happens only in expression mode (see [02.1-tokenizer.md](02.1-tokenizer.md) §6.4).\n") + w("| # | Spelling | TokenKind | Static flags of the kind |") + w("|---:|---|---|---|") + for i, (t, k) in enumerate(zip(op_text, op_kind), 1): + w(f"| {i} | `-{esc(t)}` | `{k}` | {esc(flag_of.get(k, '?').replace('TokenFlags.', ''))} |") + w(f"\nDash-operator rows: **{len(op_text)}** (`GENERATED`).") + + ch = enums["CharTraits"] + w("\n## 7. CharTraits (character-class bits) [SPEC]\n") + w("| Name | Value | Source line |") + w("|---|---|---:|") + for m in ch["members"]: + w(f"| `{m['name']}` | `{m['value']}` | L{m['line']} |") + + ns = enums["NumberSuffixFlags"] + w("\n## 8. NumberSuffixFlags and NumberFormat [SPEC]\n") + w("| Name | Value | Source line |") + w("|---|---|---:|") + for m in ns["members"]: + w(f"| `{m['name']}` | `{m['value']}` | L{m['line']} |") + if "NumberFormat" in enums: + w("") + w("| NumberFormat | Value | Source line |") + w("|---|---|---:|") + for m in enums["NumberFormat"]["members"]: + w(f"| `{m['name']}` | `{m['value']}` | L{m['line']} |") + + tm = enums["TokenizerMode"] + w("\n## 9. TokenizerMode [SPEC]\n") + w("| Name | Source line |") + w("|---|---:|") + for m in tm["members"]: + w(f"| `{m['name']}` | L{m['line']} |") + + w("\n## 10. Cross-document links\n") + w("Narrative, mode-transition table and lexical rules: [02.1-tokenizer.md](02.1-tokenizer.md). Parser consumers of each kind: [02.2-parser.md](02.2-parser.md) (planned, T22). Token subclasses: [02.1-tokenizer.md](02.1-tokenizer.md) §4.\n") + w("\nVerification: row counts asserted equal to the `TokenKind` value range (%d) and to `data/numbers.json` keys `TOKENKIND_MEMBERS`, `TOKENFLAGS_MEMBERS`; keyword and operator arrays asserted pairwise equal length (`gen_T21.py`). Static-flag and text arrays were also read in the source windows recorded in `.work/notes/` (token.cs W3-W4).\n" % nvals) + + os.makedirs(os.path.join(OUT, "02-language"), exist_ok=True) + p = os.path.join(OUT, "02-language", "TOKEN-CATALOG.md") + open(p, "w", encoding="utf-8").write("\n".join(L) + "\n") + print("wrote", p, "kinds", nvals, "kw", len(kw_text), "ops", len(op_text)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T22.py b/docs-out/tools/gen_T22.py new file mode 100644 index 0000000..9a4a5ff --- /dev/null +++ b/docs-out/tools/gen_T22.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""gen_T22.py - generated parts of the parser documents (task T22, rule 9). + +Purpose : (1) 02-language/ERROR-CATALOG.md - every ParserStrings.resx key with message text and + the places that report it (Parser.cs, tokenizer.cs, other files); + (2) fills the marked regions and in + 02-language/02.2-parser.md (production table, binary-operator precedence table). +Inputs : data/resources.json, data/parser-methods.json, data/tables.json, data/enums.json, + src/System.Management.Automation/resources/ParserStrings.resx, Parser.cs (read-only, + to classify ReportError vs ReportIncompleteInput and to find AST constructors/token kinds per method). +Outputs : docs-out/02-language/ERROR-CATALOG.md, regions of docs-out/02-language/02.2-parser.md +Run : python3 docs-out/tools/gen_T22.py (from the repo root; rerunnable) + python3 docs-out/tools/gen_T22.py --count parserstrings_keys|parser_methods|binary_operators +Label : APPROX (regex extraction; names and counts are cross-checked by reading, see documents). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys +import xml.etree.ElementTree as ET + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +PARSER = "src/System.Management.Automation/engine/parser/Parser.cs" +TOKENIZER = "src/System.Management.Automation/engine/parser/tokenizer.cs" +TOKEN = "src/System.Management.Automation/engine/parser/token.cs" +RESX = "src/System.Management.Automation/resources/ParserStrings.resx" + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def lines(p): + return open(os.path.join(ROOT, p), encoding="utf-8-sig").read().split("\n") + + +def esc(s): + return s.replace("|", "\\|").replace("\n", " ").replace("\r", "") + + +def resx_messages(): + t = ET.parse(os.path.join(ROOT, RESX)) + return {d.get("name"): (d.findtext("value") or "") for d in t.getroot().findall("data")} + + +def classify(src, line): + """E/I/? : nearest ReportError( / ReportIncompleteInput( call at or before `line` (within 14 lines).""" + for i in range(line - 1, max(line - 16, 0), -1): + s = src[i] + if "ReportIncompleteInput(" in s: + return "I" + if "ReportError(" in s: + return "E" + return "?" + + +def uses_for(key, rdata): + for k in rdata["keys"]: + if k["key"] == key: + return k["uses"] + return [] + + +def compact(lns): + out = [] + for n in sorted(set(lns)): + if out and n - out[-1] <= 2: + continue + out.append(n) + return out + + +def error_catalog(): + res = J("resources.json") + rd = res[RESX] + msgs = resx_messages() + psrc = lines(PARSER) + tsrc = lines(TOKENIZER) + rows_a, rows_b = [], [] + for k in sorted(rd["keys"], key=lambda x: x["key"]): + key = k["key"] + pl = [u["line"] for u in k["uses"] if u["file"] == PARSER] + tl = [u["line"] for u in k["uses"] if u["file"] == TOKENIZER] + other = sorted({u["file"].replace("src/System.Management.Automation/", "") for u in k["uses"] if u["file"] not in (PARSER, TOKENIZER)}) + pc = " ".join(f"L{n}({classify(psrc, n)})" for n in compact(pl)) + tc = " ".join(f"L{n}({classify(tsrc, n)})" for n in compact(tl)) + msg = esc(msgs.get(key, "(no message in resx)")) + if pl or tl: + rows_a.append((key, msg, pc, tc, ", ".join(f"`{o}`" for o in other[:3]) + (f" +{len(other)-3}" if len(other) > 3 else ""))) + else: + rows_b.append((key, msg, k["use_count"], ", ".join(f"`{o}`" for o in other[:3]) + (f" +{len(other)-3}" if len(other) > 3 else ""))) + n_keys = len(rd["keys"]) + L = [] + w = L.append + w("""--- +id: LANG-ERRCAT-001 +title: Parser and tokenizer error catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [CF-ERR-002] +sources: [src/System.Management.Automation/resources/ParserStrings.resx, src/System.Management.Automation/engine/parser/Parser.cs, src/System.Management.Automation/engine/parser/tokenizer.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/resources/_folder.md] +generated_from: [data/resources.json, data/parser-methods.json] +--- + +# Parser and tokenizer error catalog + +> STATIC-ONLY — behavior not run-verified. Every row is `GENERATED` by `docs-out/tools/gen_T22.py` (`APPROX`: use sites come from `data/resources.json`, the (E)/(I) classification looks at the nearest preceding `ReportError(` / `ReportIncompleteInput(` call in the source). Narrative: [02.2-parser.md](02.2-parser.md) §9, [02.1-tokenizer.md](02.1-tokenizer.md) §9. +""") + w("## 1. Purpose and scope [SPEC]\n") + w(f"""Catalog of the parse error IDs. An error ID is the resource key of `ParserStrings.resx` (the code passes `nameof(ParserStrings.Key)` as ID and `ParserStrings.Key` as message template; `FACT-SRC`, `{PARSER}::Parser.ReportError@L8143-L8166`). `ParserStrings.resx` holds **{n_keys}** keys; {len(rows_a)} are reported from `Parser.cs` or `tokenizer.cs` (section 3); the others are reported by semantic checks, type/class building or other code (section 4, owned by [02.4-semantic-analysis.md](02.4-semantic-analysis.md), planned T24). + +Report kinds: **(E)** `ReportError` — a plain error; **(I)** `ReportIncompleteInput` — the error sets `ParseError.IncompleteInput` when the error position is at the end of the script (so a REPL can ask for another line) (`{PARSER}::Parser.ReportIncompleteInput@L8111-L8141`); **(?)** the classifier found neither call nearby (error built through a variable, e.g. `errorId`/`errorMsg`). + +Dedup: `SaveError` drops an error when an earlier error has the same ID and the same start/end line and column; there is no cap on the number of errors (`{PARSER}::Parser.SaveError@L8026-L8057`). +""") + w("**Parse-error limits and recovery rule.** The parser never throws for a syntax error: each error is appended to `Parser.ErrorList` after de-duplication (an error is dropped only if an earlier error has the same ID and the same start and end line and column), there is no limit on the number of errors, and parsing continues by returning partial or error nodes (`ErrorStatementAst`, `ErrorExpressionAst`), by pretending a missing closing token was present, or by skipping tokens to a synchronization token (`SyncOnError`, which tracks nesting of `()`, `{}` and `[]` and never consumes end of input); `ParseError.IncompleteInput` is true only when the error was reported with `ReportIncompleteInput` and the error position is at the end of the script apart from whitespace and comments. The only exceptions are an exhausted execution stack (the whole parse is retried once on a new thread, then `ScriptTooComplicated` is reported) and any other exception during `ParseFile`/`ParseInput`, which is wrapped as a `ParseException` with `UnrecoverableParserError`.\n") + w("## 2. Source map\n") + w("| path | type | role |\n|---|---|---|") + w(f"| `{RESX}` | resx | error message templates |") + w(f"| `{PARSER}::Parser.ReportError@L8143-L8166` | method | report a plain error |") + w(f"| `{PARSER}::Parser.ReportIncompleteInput@L8111-L8141` | method | report an error that may mean 'more input expected' |") + w(f"| `{PARSER}::ParseError@L8193-L8241` | class | public error object: `Extent`, `ErrorId`, `Message`, `IncompleteInput` |\n") + w("## 3. Keys reported by Parser.cs or tokenizer.cs\n") + w("| Error ID (resource key) | Message template | Parser.cs lines | tokenizer.cs lines | Also used in |") + w("|---|---|---|---|---|") + for key, msg, pc, tc, other in rows_a: + w(f"| `{key}` | {msg} | {pc} | {tc} | {other} |") + w(f"\nRows: **{len(rows_a)}** (`GENERATED`).\n") + w("## 4. Keys used only elsewhere (semantic checks, type building, runtime)\n") + w("| Resource key | Message template | Use count | Files (first three) |") + w("|---|---|---:|---|") + for key, msg, cnt, other in rows_b: + w(f"| `{key}` | {msg} | {cnt} | {other} |") + w(f"\nRows: **{len(rows_b)}** (`GENERATED`). Keys with use count 0 have no reference found by the dataset scan (dead or referenced reflectively: `UNVERIFIED`).\n") + w("## 5. Cross-checks\n") + w("`ParserStrings` is not the only resource class a parser error ID can come from: the debug-only check `AssertErrorIdCorrespondsToMsgString` looks the ID up in `ParserStrings`, `DiscoveryExceptions`, `ExtendedTypeSystem`, `MshSnapInCmdletResources` and `ParameterBinderStrings` (`FACT-SRC`, `" + PARSER + "::Parser.AssertErrorIdCorrespondsToMsgString@L8065-L8093`). Only `ParserStrings` is cataloged here.\n") + w(f"Verification: key count taken from `data/resources.json` (`ParserStrings.resx`, key_count {n_keys}) and reconciled with the `` element count of the resx parsed with `xml.etree` (`{len(msgs)}`); every row has a message from the resx.\n") + open(os.path.join(OUT, "02-language", "ERROR-CATALOG.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + print("ERROR-CATALOG: keys", n_keys, "parser/tokenizer", len(rows_a), "others", len(rows_b), "resx", len(msgs)) + return n_keys, len(rows_a) + + +def productions(): + pm = [x for x in J("parser-methods.json") if x["file"] == PARSER] + src = lines(PARSER) + out = ["| Production (method) | Kind | File@Lines | Tokens referenced (`TokenKind`, APPROX) | AST created (`new …Ast(`, APPROX) | Error IDs (`ParserStrings`, APPROX) |", + "|---|---|---|---|---|---|"] + for m in pm: + body = "\n".join(src[m["start"] - 1:m["end"]]) + toks = sorted(set(re.findall(r"TokenKind\.(\w+)", body))) + asts = sorted(set(re.findall(r"new\s+(\w+Ast)\s*\(", body))) + errs = sorted(set(re.findall(r"nameof\(ParserStrings\.(\w+)\)", body))) + tname = m["type"].split(".")[-1] + kind = "rule" if m["method"].endswith("Rule") else "helper" + def cell(v, cap=14): + if not v: + return "—" + s = ", ".join(f"`{x}`" for x in v[:cap]) + return s + (f" +{len(v)-cap}" if len(v) > cap else "") + out.append(f"| `{m['method']}` | {kind} | `{PARSER}::{tname}.{m['method']}@L{m['start']}-L{m['end']}` | {cell(toks)} | {cell(asts, 8)} | {cell(errs, 8)} |") + return out, len(pm) + + +def precedence(): + """binary operators: token kind -> precedence level (TokenFlags.BinaryPrecedence*) - GENERATED from token.cs tables.""" + sys.path.insert(0, os.path.join(OUT, "tools")) + import gen_T21 as g + tb = J("tables.json") + en = {e["full_name"].split(".")[-1]: e for e in J("enums.json") if e["file"] == TOKEN} + flags = {m["name"]: m["value"] for m in en["TokenFlags"]["members"]} + kinds = en["TokenKind"]["members"] + sfl = [re.sub(r"\s+", " ", x) for x in g.split_top(g.strip_comments(tb[f"{TOKEN}::s_staticTokenFlags"]["text"]))] + txt = [g.unq(x) for x in g.split_top(g.strip_comments(tb[f"{TOKEN}::s_tokenText"]["text"]))] + prec = {k: int(v, 16) for k, v in flags.items() if k.startswith("BinaryPrecedence") and k != "BinaryPrecedenceMask"} + rows = [] + for m in kinds: + v = int(m["value"]) + f = sfl[v] + if "BinaryOperator" not in f: + continue + p = [k for k in prec if "TokenFlags." + k in f] + rows.append((prec[p[0]] if p else -1, p[0].replace("BinaryPrecedence", "") if p else "?", m["name"], txt[v], "Unary" if "UnaryOperator" in f else "", "CanConstantFold" if "CanConstantFold" in f else "", "DisallowedInRestrictedMode" if "DisallowedInRestrictedMode" in f else "")) + rows.sort(key=lambda r: (r[0], r[2])) + out = ["| Level (value) | Level name | TokenKind | Text | Also unary | Constant-folded | Restricted-mode flag |", + "|---:|---|---|---|---|---|---|"] + for r in rows: + out.append(f"| {r[0]} | {r[1]} | `{r[2]}` | `{esc(r[3])}` | {r[4]} | {r[5]} | {r[6]} |") + return out, len(rows) + + +def fill(doc, tag, rows): + p = os.path.join(OUT, "02-language", doc) + t = open(p, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()" % (tag, tag), re.S) + if not pat.search(t): + print("region missing", tag, doc) + return + t = pat.sub(lambda m: m.group(1) + "\n" + "\n".join(rows) + "\n" + m.group(2), t) + open(p, "w", encoding="utf-8").write(t) + + +def main(): + if len(sys.argv) > 2 and sys.argv[1] == "--count": + what = sys.argv[2] + if what == "parserstrings_keys": + print(J("resources.json")[RESX]["key_count"]) + elif what == "parser_methods": + print(len([x for x in J("parser-methods.json") if x["file"] == PARSER])) + elif what == "binary_operators": + print(precedence()[1]) + elif what == "parserstrings_keys_parser_tokenizer": + rd = J("resources.json")[RESX] + print(sum(1 for k in rd["keys"] if any(u["file"] in (PARSER, TOKENIZER) for u in k["uses"]))) + return 0 + error_catalog() + if os.path.exists(os.path.join(OUT, "02-language", "02.2-parser.md")): + rows, n = productions() + fill("02.2-parser.md", "productions", rows) + rows, n2 = precedence() + fill("02.2-parser.md", "precedence", rows) + print("productions", n, "binary operators", n2) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T23.py b/docs-out/tools/gen_T23.py new file mode 100644 index 0000000..23d1bef --- /dev/null +++ b/docs-out/tools/gen_T23.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""gen_T23.py - render 02-language/AST-CATALOG.md from data/*.json (task T23, rule 9). + +Purpose : node catalog `NodeType | Base | Children | Created by | Consumed by`, visitor method tables, + inheritance diagram, enums declared in ast.cs. Never hand-typed. +Inputs : data/ast-hierarchy.json, data/members-tierA.json, data/visitors.json, data/enums.json, + src/System.Management.Automation/engine/parser/ast.cs (read-only, InternalVisit bodies). +Outputs : docs-out/02-language/AST-CATALOG.md +Run : python3 docs-out/tools/gen_T23.py (from the repo root) + python3 docs-out/tools/gen_T23.py --count ast_classes|abstract|iv1|iv2|children_gaps +Label : APPROX. "Children" are the receivers of `.InternalVisit(` calls in the node's InternalVisit method, + in source order (a raw expression such as `ifClause.Item1` is shown as written). "Created by" is + `new XAst(` in Parser.cs (data/ast-hierarchy.json). "Consumed by" lists visitor classes defining Visit. +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +ASTCS = "src/System.Management.Automation/engine/parser/ast.cs" +DEFAULT_VISITORS = {"AstVisitor", "AstVisitor2", "ICustomAstVisitor", "ICustomAstVisitor2", "DefaultCustomAstVisitor", "DefaultCustomAstVisitor2"} + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def short(n): + return n.split(".")[-1] + + +def nodes(): + h = [x for x in J("ast-hierarchy.json") if x["file"] == ASTCS] + mem = {short(t["full_name"]): t for t in J("members-tierA.json") if t["file"] == ASTCS} + return h, mem + + +def children(src, tmem): + if not tmem: + return None, False + iv = [m for m in tmem["members"] if m["name"] in ("InternalVisit", "InternalVisitChildren") and m.get("start")] + out, v2 = [], False + for m in iv: + body = "\n".join(src[m["start"] - 1:m["end"]]) + if "AstVisitor2" in body: + v2 = True + ev = [] # (position, kind, a, b) + for r in re.finditer(r"foreach\s*\(\s*(?:var|[\w<>,\.]+)\s+(\w+)\s+in\s+([\w\.]+)\s*\)", body): + ev.append((r.start(), "loop", r.group(1), r.group(2) + "[]")) + for r in re.finditer(r"(?:var|\w+)\s+(\w+)\s*=\s*([\w\.]+)\[\w+\];", body): + ev.append((r.start(), "loop", r.group(1), r.group(2) + "[]")) + for r in re.finditer(r"VisitAndShallContinue\((\w+)\)\s*(?:&&|;)", body): + ev.append((r.start(), "direct", r.group(1), None)) + for r in re.finditer(r"([A-Za-z_][\w\.\[\]]*)\.InternalVisit\(", body): + ev.append((r.start(), "visit", r.group(1), None)) + for r in re.finditer(r"StatementBlockAst\.InternalVisit\(visitor,\s*(\w+),\s*(\w+)", body): + ev.append((r.start(), "direct", f"{r.group(1)}, {r.group(2)} (static helper)", None)) + ev.sort() + lv = {} + local_fn = "VisitAndShallContinue" in body + for _, kind, x, y in ev: + if kind == "loop": + lv[x] = y + elif kind == "direct": + out.append(x) + else: + if x in ("visitor", "StatementBlockAst") or (local_fn and x == "ast"): + continue + root = re.split(r"[\.\[]", x, maxsplit=1) + if root[0] in lv: + x = lv[root[0]] + (x[len(root[0]):] if len(root) > 1 else "") + out.append(x) + return out, v2 + + +def main(): + h, mem = nodes() + vis = J("visitors.json") + en = [e for e in J("enums.json") if e["file"] == ASTCS] + if len(sys.argv) > 2 and sys.argv[1] == "--count": + w = sys.argv[2] + if w == "ast_classes": + print(len(h)) + elif w == "abstract": + print(sum(1 for x in h if "abstract" in x["modifiers"])) + elif w in ("iv1", "iv2"): + n = "System.Management.Automation.Language." + ("ICustomAstVisitor" if w == "iv1" else "ICustomAstVisitor2") + print([v for v in vis if v["name"] == n][0]["visit_method_count"]) + return 0 + src = open(os.path.join(ROOT, ASTCS), encoding="utf-8-sig").read().split("\n") + byname = {short(x["name"]): x for x in h} + # visitor index: Visit -> visitor classes (non-default) + vindex = {} + for v in vis: + s = short(v["name"]) + if s in DEFAULT_VISITORS: + continue + for m in v["visit_methods"]: + vindex.setdefault(m, set()).add(s) + L = [] + w = L.append + nabs = sum(1 for x in h if "abstract" in x["modifiers"]) + w("""--- +id: LANG-ASTCAT-001 +title: AST node catalog +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/ast.cs, src/System.Management.Automation/engine/parser/AstVisitor.cs, src/System.Management.Automation/engine/parser/PreOrderVisitor.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/ast-hierarchy.json, data/members-tierA.json, data/visitors.json, data/enums.json] +--- + +# AST node catalog + +> STATIC-ONLY — behavior not run-verified. Every table is `GENERATED` by `docs-out/tools/gen_T23.py` (`APPROX`: regex extraction, see the column notes). Narrative: [02.3-ast.md](02.3-ast.md). +""") + w("## 1. Purpose and scope [SPEC]\n") + w(f"""Complete list of the AST classes declared in `ast.cs` ({len(h)} classes, {nabs} of them abstract), with base class, visit-order children, parser productions that create them, and the visitors that handle them; the two visitor interface method lists; the inheritance diagram; and the enums declared in `ast.cs`. Count source: `data/ast-hierarchy.json` filtered to `ast.cs` (the dataset has one more row from `VariableAnalysis.cs`, a private helper class, which is excluded). +""") + w("## 2. Source map\n") + w("| path | type | role |\n|---|---|---|") + w(f"| `{ASTCS}::Ast@L91-L420` | abstract class | root of all nodes |") + w("| `src/System.Management.Automation/engine/parser/AstVisitor.cs` | file | `ICustomAstVisitor(2)`, `DefaultCustomAstVisitor(2)`, `AstSearcher`, DEBUG checkers |") + w("| `src/System.Management.Automation/engine/parser/PreOrderVisitor.cs` | file | `AstVisitAction`, `AstVisitor(2)`, `IAstPostVisitHandler` |\n") + w("## 3. Node catalog\n") + w("Column notes: **Base** = direct base class; **Children (visit order)** = receivers of `.InternalVisit(` in the node's `InternalVisit` method, in source order (`—` = leaf or no `InternalVisit`); **v2** = the method mentions `AstVisitor2`, i.e. visiting is gated or extended for second-generation visitors; **Created by** = parser methods containing `new (` (`—` = not created by `Parser.cs`: abstract, compiler-synthesized or created in other files); **Consumed by** = non-default visitor classes that define `Visit` (first six, then `+N`).\n") + w("| Node | Kind | Base | Children (visit order, APPROX) | v2 | Created by (Parser.cs) | Consumed by (APPROX) | Source |") + w("|---|---|---|---|---|---|---|---|") + gaps = 0 + for x in sorted(h, key=lambda z: z["start"]): + n = short(x["name"]) + base = x["base_chain"][0] if x["base_chain"] else "—" + ch, v2 = children(src, mem.get(n)) + if ch is None or not ch: + chs = "—" + if "abstract" not in x["modifiers"]: + gaps += 1 + else: + chs = ", ".join(f"`{c}`" for c in ch) + cb = sorted({m["method"] for m in x["created_by_parser"]}) + cbs = ", ".join(f"`{c}`" for c in cb) if cb else "—" + vn = "Visit" + n[:-3] if n.endswith("Ast") else "" + cons = sorted(vindex.get(vn, [])) + cs = ", ".join(f"`{c}`" for c in cons[:6]) + (f" +{len(cons)-6}" if len(cons) > 6 else "") if cons else "—" + kind = "abstract" if "abstract" in x["modifiers"] else ("internal" if x["accessibility"] != "public" else "concrete") + cond = f" `#if {'/'.join(x['cond'])}`" if x["cond"] else "" + w(f"| `{n}` | {kind}{cond} | `{base}` | {chs} | {'yes' if v2 else ''} | {cbs} | {cs} | `{ASTCS}::{n}@L{x['start']}-L{x['end']}` |") + w(f"\nRows: **{len(h)}** (`GENERATED`). Concrete nodes with no recognized child visit: {gaps} (leaves, or children visited through a different construct).\n") + w("## 4. Inheritance diagram\n") + w("```mermaid\nflowchart LR") + for x in sorted(h, key=lambda z: z["start"]): + n = short(x["name"]) + if x["base_chain"]: + w(f" {x['base_chain'][0]} --> {n}") + w("```\n") + w("## 5. Visitor interfaces\n") + w("Method lists are the `Visit*` methods found in each type (`GENERATED`, `APPROX`). `ICustomAstVisitor` returns `object`; `AstVisitor` returns `AstVisitAction`.\n") + for want in ("ICustomAstVisitor", "ICustomAstVisitor2", "AstVisitor", "AstVisitor2"): + v = [q for q in vis if short(q["name"]) == want][0] + w(f"**{want}** (`{v['file']}::{want}@L{v['start']}-L{v['end']}`, {v['visit_method_count']} visit methods): " + ", ".join(f"`{m}`" for m in v["visit_methods"]) + "\n") + w("## 6. Visitor implementations in the repository\n") + w("| Class | Bases | Visit methods defined | File |\n|---|---|---:|---|") + for q in sorted(vis, key=lambda z: z["name"]): + w(f"| `{short(q['name'])}` | {', '.join('`'+b+'`' for b in q['bases']) or '—'} | {q['visit_method_count']} | `{q['file']}` |") + w("") + w("## 7. Enums declared in ast.cs\n") + for e in sorted(en, key=lambda z: z["start"]): + mem_s = ", ".join(f"`{m['name']}`" + (f"={m['value']}" if m["value"] not in (None, "") else "") for m in e["members"]) + w(f"- `{short(e['full_name'])}` (`{ASTCS}::{short(e['full_name'])}@L{e['start']}-L{e['end']}`): {mem_s}") + w("") + w("""## 8. Non-AST public types in ast.cs + +Type names (`ITypeName` family), `ScriptRequirements` and `CommentHelpInfo` live in `ast.cs` but are not `Ast` nodes (read in notes, `FACT-SRC`): see [02.3-ast.md](02.3-ast.md) §5. + +Verification: node rows reconciled with `data/ast-hierarchy.json` (filtered by file); `InternalVisit` ranges come from `data/members-tierA.json`; edges in the diagram are the direct base of each row; visitor lists reconcile with `data/visitors.json`. +""") + open(os.path.join(OUT, "02-language", "AST-CATALOG.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + print("AST-CATALOG: nodes", len(h), "abstract", nabs, "gaps", gaps) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T24.py b/docs-out/tools/gen_T24.py new file mode 100644 index 0000000..b2ee873 --- /dev/null +++ b/docs-out/tools/gen_T24.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""gen_T24.py - generated check table of 02-language/02.4-semantic-analysis.md (task T24, rule 9). + +Purpose : table `Check (member) | Where | Error ID | Trigger (message template) | Runs when` for every + `nameof(ParserStrings.X)` site in SemanticChecks.cs, SymbolResolver.cs and PSType.cs. +Inputs : data/types.json, data/members-tierA.json, ParserStrings.resx, the three source files (read-only). +Outputs : fills region ... in docs-out/02-language/02.4-semantic-analysis.md +Run : python3 docs-out/tools/gen_T24.py (from the repo root) + python3 docs-out/tools/gen_T24.py --count sites|ids +Label : APPROX (regex over source; the enclosing member is the innermost member whose line range holds the site). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys +import xml.etree.ElementTree as ET + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +S = "src/System.Management.Automation/engine/parser/" +RESX = "src/System.Management.Automation/resources/ParserStrings.resx" +FILES = { + S + "SymbolResolver.cs": "post-parse, symbol resolution (before semantic checks)", + S + "SemanticChecks.cs": None, + S + "PSType.cs": "compile of the root script block (class/enum definition)", +} +STAGE = { + "SemanticChecks": "post-parse semantic checks (`PerformPostParseChecks`)", + "RestrictedLanguageChecker": "`data` section semantic check; module-manifest / restricted evaluation; runtime check of data sections", + "DscResourceChecker": "post-parse, only for classes with `[DscResource()]`", +} + + +def J(n): + return json.load(open(os.path.join(DATA, n))) + + +def esc(s): + return s.replace("|", "\\|").replace("\n", " ").replace("\r", "") + + +def sites(): + types = J("types.json") + members = {(m["file"], m["full_name"]): m["members"] for m in J("members-tierA.json")} + msgs = {d.get("name"): (d.findtext("value") or "") for d in ET.parse(os.path.join(ROOT, RESX)).getroot().findall("data")} + rows = [] + for f, stage in FILES.items(): + src = open(os.path.join(ROOT, f), encoding="utf-8-sig").read().split("\n") + ts = [t for t in types if t["file"] == f] + for i, l in enumerate(src, 1): + for m in re.finditer(r"nameof\(ParserStrings\.(\w+)\)", l): + inner = [t for t in ts if t["start"] <= i <= t["end"]] + inner.sort(key=lambda t: t["end"] - t["start"]) + tn = inner[0]["full_name"].split("Language.")[-1] if inner else "?" + mem = "(type level)" + if inner: + mm = [x for x in members.get((f, inner[0]["full_name"]), []) if x.get("start") and x["start"] <= i <= x["end"]] + mm.sort(key=lambda x: x["end"] - x["start"]) + if mm: + mem = mm[0]["name"] + short = tn.split(".")[0] + rows.append((f, tn, mem, i, m.group(1), msgs.get(m.group(1), "(no message)"), stage or STAGE.get(short, "?"))) + return rows + + +def main(): + rows = sites() + if len(sys.argv) > 2 and sys.argv[1] == "--count": + print(len(rows) if sys.argv[2] == "sites" else len({r[4] for r in rows})) + return 0 + out = ["| Check (member) | Where | Error ID | Trigger (message template) | Runs when |", "|---|---|---|---|---|"] + for f, tn, mem, i, eid, msg, stage in rows: + out.append(f"| `{tn}.{mem}` | `{f}@L{i}` | `{eid}` | {esc(msg)} | {stage} |") + p = os.path.join(OUT, "02-language", "02.4-semantic-analysis.md") + t = open(p, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()", re.S) + if not pat.search(t): + print("region missing") + return 1 + t = pat.sub(lambda m: m.group(1) + "\n" + "\n".join(out) + "\n" + m.group(2), t) + open(p, "w", encoding="utf-8").write(t) + print("semantic check sites", len(rows), "distinct ids", len({r[4] for r in rows})) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T25.py b/docs-out/tools/gen_T25.py new file mode 100644 index 0000000..fd0fda6 --- /dev/null +++ b/docs-out/tools/gen_T25.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""gen_T25.py - render 02-language/COMPILATION-MAP.md (task T25, rule 9). + +Purpose : table `NodeType -> how it is compiled`: every Compiler.Visit* method with its line range + (from data/members-tierA.json) joined with hand annotations (tools/annotations/T25-compilation-map.json), + plus the list of Compiler helper methods with line ranges. +Inputs : data/members-tierA.json, data/types.json, tools/annotations/T25-compilation-map.json +Outputs : docs-out/02-language/COMPILATION-MAP.md +Run : python3 docs-out/tools/gen_T25.py (from the repo root) + python3 docs-out/tools/gen_T25.py --count visits|helpers +Label : ranges APPROX (scanner); annotations FACT-SRC (read in Compiler.cs, notes in .work/notes). +Language: Python 3 (D-002) +""" +import json +import os +import re +import sys +from collections import Counter + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") +F = "src/System.Management.Automation/engine/parser/Compiler.cs" + + +def esc(s): + return s.replace("|", "\\|") + + +SUPPORT = { + "Constant": "yes", "Call": "yes (by-ref or value-type instance call forces full JIT)", "Block": "yes (defines locals)", "Assign": "yes (local, parameter, index, member; not compound)", + "Dynamic": "yes (DynamicInstruction<...> per site)", "Label": "yes", "Variable": "yes", "NewArrayInit": "yes", "IfThen": "yes", "Catch": "yes (no filters)", + "Field": "yes (literal/static/instance via reflection)", "TryFinally": "yes", "Property": "yes (getter call)", "New": "yes (by-ref ctor forces JIT)", "Not": "yes (bool only)", + "Goto": "yes", "Throw": "yes", "Parameter": "yes", "Convert": "numeric convert, otherwise no-op", "Rethrow": "yes", "TryCatch": "yes", "SwitchCase": "int constants only", + "NotEqual": "numeric/reference", "Condition": "yes", "TryCatchFinally": "yes", "Equal": "numeric/reference", "Switch": "int test values, no comparison method", + "IfThenElse": "yes", "TypeIs": "yes", "Empty": "yes", "AndAlso": "bool only", "SymbolDocument": "n/a (debug info document)", "Subtract": "yes", "Return": "yes (goto)", + "OrElse": "bool only", "Multiply": "yes", "Lambda": "nested lambdas via closure", "GreaterThan": "numeric", "Divide": "yes", "DebugInfo": "recorded only", "Continue": "yes (goto)", + "Break": "yes (goto)", "ArrayAccess": "yes (indexer)", "Add": "yes", +} + + +def irnodes(): + src = open(os.path.join(ROOT, F), encoding="utf-8-sig").read() + cnt = Counter(re.findall(r"Expression\.([A-Z][A-Za-z]*)\(", src)) + rows = ["| LINQ factory | Calls in Compiler.cs | Interpreter (LightCompiler) support |", "|---|---:|---|"] + for k, v in sorted(cnt.items(), key=lambda kv: (-kv[1], kv[0])): + rows.append(f"| `Expression.{k}` | {v} | {SUPPORT.get(k, '?')} |") + return rows, len(cnt) + + +def fill_region(doc, tag, rows): + p = os.path.join(OUT, "02-language", doc) + if not os.path.exists(p): + return + t = open(p, encoding="utf-8").read() + pat = re.compile(r"()\n?.*?\n?()" % (tag, tag), re.S) + if pat.search(t): + open(p, "w", encoding="utf-8").write(pat.sub(lambda m: m.group(1) + "\n" + "\n".join(rows) + "\n" + m.group(2), t)) + + +def main(): + mem = json.load(open(os.path.join(DATA, "members-tierA.json"))) + c = [x for x in mem if x["file"] == F and x["full_name"].endswith(".Compiler")][0] + ann = json.load(open(os.path.join(OUT, "tools", "annotations", "T25-compilation-map.json")))["visits"] + visits = [m for m in c["members"] if m["name"].startswith("Visit")] + helpers = [m for m in c["members"] if m.get("kind") == "method" and not m["name"].startswith("Visit") and m.get("start")] + if len(sys.argv) > 2 and sys.argv[1] == "--count": + print(len(visits) if sys.argv[2] == "visits" else len(helpers) if sys.argv[2] == "helpers" else irnodes()[1]) + return 0 + missing = [v["name"] for v in visits if v["name"] not in ann] + extra = [k for k in ann if k not in {v["name"] for v in visits}] + assert not missing and not extra, (missing, extra) + L = [] + w = L.append + w("""--- +id: LANG-COMPMAP-001 +title: Compilation map +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/parser/Compiler.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/parser/_folder.md] +generated_from: [data/members-tierA.json] +--- + +# Compilation map + +> STATIC-ONLY — behavior not run-verified. Rows are `GENERATED` by `docs-out/tools/gen_T25.py`: method names and line ranges from `data/members-tierA.json` (`APPROX`), the "compiled form" text from the hand-written annotation file `docs-out/tools/annotations/T25-compilation-map.json` (`FACT-SRC`, read in `Compiler.cs`). Narrative: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md). Intermediate representation: [IR-SPEC.md](IR-SPEC.md). +""") + w("## 1. Purpose and scope [SPEC]\n") + w(f"""For every AST node type, how `Compiler` (an `ICustomAstVisitor2`) turns it into a LINQ expression tree. `Compiler` has {len(visits)} `Visit*` methods (one per visitor method of `ICustomAstVisitor` plus `ICustomAstVisitor2`, see [AST-CATALOG.md](AST-CATALOG.md) §5) and {len(helpers)} other methods (section 4). +""") + w("## 2. Source map\n") + w(f"| path | type | role |\n|---|---|---|\n| `{F}::Compiler@L{c['members'][0]['start']}-L{c['members'][-1]['end']}` | class | AST → expression-tree compiler |\n") + w("## 3. Node → compiled form\n") + w("| AST node (Visit method) | Compiled form | How (FACT-SRC) | Source |") + w("|---|---|---|---|") + for v in visits: + k, t = ann[v["name"]] + w(f"| `{v['name'][5:]}Ast` (`{v['name']}`) | {k} | {esc(t)} | `{F}::Compiler.{v['name']}@L{v['start']}-L{v['end']}` |") + w(f"\nRows: **{len(visits)}** (`GENERATED`). \"not visited\" rows are asserted unreachable in the compiler (their parents handle them).\n") + w("## 4. Helper methods of Compiler\n") + w("| Method | Source |\n|---|---|") + seen = {} + for h in helpers: + n = seen.get(h["name"], 0) + seen[h["name"]] = n + 1 + w(f"| `{h['name']}`{' (overload)' if n else ''} | `{F}::Compiler.{h['name']}@L{h['start']}-L{h['end']}` |") + w("\nVerification: all 63 `Visit*` methods were read (notes W3-W21); the table is joined by name and the generator asserts that every `Visit*` method has an annotation and vice versa; `VisitPipeline`, `VisitTrap`, `VisitHashtable`, `VisitCatchClause`, `VisitDoWhileStatement` were re-read for this catalog.\n") + open(os.path.join(OUT, "02-language", "COMPILATION-MAP.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + rows, nk = irnodes() + fill_region("IR-SPEC.md", "irnodes", rows) + print("COMPILATION-MAP: visits", len(visits), "helpers", len(helpers), "ir factory kinds", nk) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs-out/tools/gen_T27.py b/docs-out/tools/gen_T27.py new file mode 100644 index 0000000..ae8bbd9 --- /dev/null +++ b/docs-out/tools/gen_T27.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""gen_T27.py - render 02-language/INSTRUCTION-SET.md (task T27, rule 9). + +Purpose : catalog of the interpreter's Instruction classes (data/instructions.json), grouped by file, with + kind (abstract base / concrete / nested family member), plus the DLR-style generated families that + the scanner sees as single names (arity tables), and the interpreter infrastructure types. +Inputs : data/instructions.json, data/types.json +Outputs : docs-out/02-language/INSTRUCTION-SET.md +Run : python3 docs-out/tools/gen_T27.py (from the repo root) + python3 docs-out/tools/gen_T27.py --count instructions|abstract|files +Label : APPROX (regex scan of `class X : Instruction`); stack effects are in the narrative (read by hand). +Language: Python 3 (D-002) +""" +import json +import os +import sys +from collections import OrderedDict + +ROOT = os.path.abspath(os.environ.get("REPO_ROOT", ".")) +OUT = os.path.join(ROOT, "docs-out") +DATA = os.path.join(OUT, "data") + + +def main(): + ins = json.load(open(os.path.join(DATA, "instructions.json"))) + if len(sys.argv) > 2 and sys.argv[1] == "--count": + k = sys.argv[2] + print(len(ins) if k == "instructions" else sum(1 for x in ins if "abstract" in x["modifiers"]) if k == "abstract" else len({x["file"] for x in ins})) + return 0 + groups = OrderedDict() + for x in sorted(ins, key=lambda z: (z["file"], z["start"])): + groups.setdefault(x["file"], []).append(x) + L = [] + w = L.append + w("""--- +id: LANG-INSTR-001 +title: Instruction set +kind: catalog +tier: A +status: draft +run_verification: none +scope_class: CORE +version_pin: v7.6.9 +pin_method: "last-commit-history + tree metadata: commit d4d786f message " +canonical_facts: [] +sources: [src/System.Management.Automation/engine/interpreter/Instruction.cs, src/System.Management.Automation/engine/interpreter/InstructionList.cs] +mirror_cards: [src-mirror/src/System.Management.Automation/engine/interpreter/_folder.md] +generated_from: [data/instructions.json, data/types.json] +--- + +# Instruction set + +> STATIC-ONLY — behavior not run-verified. The class list is `GENERATED` by `docs-out/tools/gen_T27.py` from `data/instructions.json` (`APPROX`: classes whose base chain reaches `Instruction`). Stack effects and semantics: [TIERING-POLICY.md](TIERING-POLICY.md) §6 and [EQUIVALENCE-SPEC.md](EQUIVALENCE-SPEC.md). Interpreter structure: [02.5-compiler-and-engine.md](02.5-compiler-and-engine.md) §7. +""") + absn = sum(1 for x in ins if "abstract" in x["modifiers"]) + w("## 1. Purpose and scope [SPEC]\n") + w(f"""The interpreter (the "light compiler" in `engine/interpreter`) executes a flat array of `Instruction` objects over an object stack. This catalog lists the {len(ins)} instruction classes the scanner found ({absn} abstract bases) in {len(groups)} files. Generic families that are expanded by arity (`DynamicInstruction<…>`, `ActionCallInstruction<…>`, `FuncCallInstruction<…>`) are listed once per file by the scanner and described in section 4. +""") + w("## 2. Source map\n") + w("| path | role |\n|---|---|") + w("| `src/System.Management.Automation/engine/interpreter/Instruction.cs` | abstract `Instruction` base, `IInstructionProvider`, `NotInstruction` |") + w("| `src/System.Management.Automation/engine/interpreter/InstructionList.cs` | `InstructionList` emitter and `InstructionArray` |") + w("| `src/System.Management.Automation/engine/interpreter/Interpreter.cs` | `Interpreter.Run` loop |\n") + w("## 3. Instruction classes by file\n") + for f, xs in groups.items(): + w(f"### `{f.split('/')[-1]}`\n") + w("| Class | Kind | Base | Source |\n|---|---|---|---|") + for x in xs: + n = x["name"].split("Interpreter.")[-1] + kind = "abstract" if "abstract" in x["modifiers"] else ("sealed" if "sealed" in x["modifiers"] else "concrete") + w(f"| `{n}` | {kind} | `{x['chain'][0] if x['chain'] else '—'}` | `{f}@L{x['start']}-L{x['end']}` |") + w("") + w("## 4. Generated families (not enumerated by the scanner)\n") + w("""| Family | Members | Evidence | +|---|---|---| +| `DynamicInstruction<…>` | 16 generic classes, 0..15 call arguments (`DynamicInstruction` … ``) | `src/System.Management.Automation/engine/interpreter/DynamicInstructions.Generated.cs` (notes W1-W2) | +| `ActionCallInstruction<…>` | 10 classes, 0..9 arguments | `src/System.Management.Automation/engine/interpreter/CallInstruction.Generated.cs` | +| `FuncCallInstruction<…>` | 10 classes, 0..9 arguments | same file | +| `GetArrayItemInstruction`, `SetArrayItemInstruction`, `TypeIsInstruction`, `TypeAsInstruction`, `DefaultValueInstruction`, `NewArrayInstruction`, `NewArrayInitInstruction` | created per element type on demand by `InstructionFactory` (15 pre-seeded types) | `src/System.Management.Automation/engine/interpreter/InstructionFactory.cs` | + +Counts for these families come from the generated-file notes (`FACT-SRC`), not from `data/instructions.json`. +""") + w("Verification: instruction class list reconciles with `data/instructions.json` (count above); abstract/sealed taken from the class modifiers; family sizes from the three generated source files read in full (notes `.work/notes/src_System.Management.Automation_engine_interpreter_*`).\n") + open(os.path.join(OUT, "02-language", "INSTRUCTION-SET.md"), "w", encoding="utf-8").write("\n".join(L) + "\n") + print("INSTRUCTION-SET: classes", len(ins), "abstract", absn, "files", len(groups)) + + +if __name__ == "__main__": + sys.exit(main())