diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 754ff1555..4b203426e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.11", "3.12"] # no particular need for 3.9 or 3.10 + python-version: ["3.8", "3.11", "3.12", "3.13", "3.14", "3.15"] os: [macos-latest, ubuntu-latest, windows-latest] steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 @@ -21,6 +21,7 @@ jobs: uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v4.6.0 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Upgrade pip run: >- python -m pip install diff --git a/README.md b/README.md index 7bb6b2277..1f3d7f54b 100644 --- a/README.md +++ b/README.md @@ -351,9 +351,21 @@ optional arguments: --binary BINARY location of binary to use for YAPF ``` -## Python features not yet supported -* Python 3.12 – [PEP 695 – Type Parameter Syntax](https://peps.python.org/pep-0695/) – [YAPF #1170](https://github.com/google/yapf/issues/1170) -* Python 3.12 – [PEP 701 – Syntactic formalization of f-strings](https://peps.python.org/pep-0701/) – [YAPF #1136](https://github.com/google/yapf/issues/1136) +## Python syntax support + +* Python 3.12 - [PEP 695 - Type parameter syntax](https://peps.python.org/pep-0695/) - **Supported.** See [implementation notes](docs/pep-0695.md). +* Python 3.13 - [PEP 696 - Type parameter defaults](https://peps.python.org/pep-0696/) - **Supported.** See [implementation notes](docs/pep-0696.md). +* Python 3.12 - [PEP 701 - F-string syntax](https://peps.python.org/pep-0701/) - **Supported.** See [implementation notes](docs/pep-0701.md). +* Python 3.14 - [PEP 750 - Template strings](https://peps.python.org/pep-0750/) - **Supported.** See [implementation notes](docs/pep-0750.md). +* Python 3.14 - [PEP 758 - Unparenthesized exception lists](https://peps.python.org/pep-0758/) - **Supported.** See [implementation notes](docs/pep-0758.md). +* Python 3.15 - [PEP 798 - Unpacking comprehensions](https://peps.python.org/pep-0798/) - **Supported.** See [implementation notes](docs/pep-0798.md). +* Python 3.15 - [PEP 810 - Explicit lazy imports](https://peps.python.org/pep-0810/) - **Supported.** See [implementation notes](docs/pep-0810.md). + +Interpolated string literals, including replacement and debug expressions, are +preserved verbatim; YAPF formats the surrounding code, not their contents. + +See [branch dependencies and validation](docs/pep-feature-branches.md) for the +feature-branch layout and test limitations. ## Knobs diff --git a/docs/pep-0695.md b/docs/pep-0695.md new file mode 100644 index 000000000..2a6bc0bbe --- /dev/null +++ b/docs/pep-0695.md @@ -0,0 +1,24 @@ +# PEP 695: Type parameter syntax + +Specification: https://peps.python.org/pep-0695/ + +Language version: Python 3.12. Branch dependency: the shared grammar-cache isolation fix. + +## Supported formatting + +Type aliases, generic classes and functions, bounds, constraints, `*Ts`, and `**P`. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep695_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-0696.md b/docs/pep-0696.md new file mode 100644 index 000000000..1395d1a8c --- /dev/null +++ b/docs/pep-0696.md @@ -0,0 +1,24 @@ +# PEP 696: Type parameter defaults + +Specification: https://peps.python.org/pep-0696/ + +Language version: Python 3.13. Branch dependency: PEP 695. + +## Supported formatting + +Defaults for type variables, variadic type variables, and parameter specifications, including assignment spacing, comments, and wrapping. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep696_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-0701.md b/docs/pep-0701.md new file mode 100644 index 000000000..7aaadcd99 --- /dev/null +++ b/docs/pep-0701.md @@ -0,0 +1,29 @@ +# PEP 701: F-string grammar + +Specification: https://peps.python.org/pep-0701/ + +Language version: Python 3.12. Branch dependency: the shared grammar-cache isolation fix. + +## Supported formatting + +Quote reuse, nesting, comments, backslashes, multiline replacement fields, token positions, and line-range formatting. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +Interpolated literals remain opaque STRING tokens. Literal text, replacement +expressions, debug-expression whitespace, and template expression metadata are +preserved verbatim. Only surrounding code is reformatted. The scanner identifies +lexical boundaries; it does not validate arbitrary replacement expressions. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep701_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-0750.md b/docs/pep-0750.md new file mode 100644 index 000000000..7b8ac4265 --- /dev/null +++ b/docs/pep-0750.md @@ -0,0 +1,29 @@ +# PEP 750: Template strings + +Specification: https://peps.python.org/pep-0750/ + +Language version: Python 3.14. Branch dependency: PEP 701. + +## Supported formatting + +All `t`/`T` and raw prefixes, every quote form, mixed f-string/t-string nesting, comments, backslashes, and multiline fields. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +Interpolated literals remain opaque STRING tokens. Literal text, replacement +expressions, debug-expression whitespace, and template expression metadata are +preserved verbatim. Only surrounding code is reformatted. The scanner identifies +lexical boundaries; it does not validate arbitrary replacement expressions. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep750_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-0758.md b/docs/pep-0758.md new file mode 100644 index 000000000..2e93f28ce --- /dev/null +++ b/docs/pep-0758.md @@ -0,0 +1,28 @@ +# PEP 758: Unparenthesized exception lists + +Specification: https://peps.python.org/pep-0758/ + +Language version: Python 3.14. Branch dependency: the shared grammar-cache isolation fix. + +## Supported formatting + +Any number of exception types for `except` and `except*`, including a trailing comma; unparenthesized lists cannot bind an `as` target. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +An unparenthesized list with no explicit continuation remains on one physical +line, even when over the column limit. No implicit parentheses or unsafe line +breaks are introduced. Use an explicitly parenthesized list to enable wrapping. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep758_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-0798.md b/docs/pep-0798.md new file mode 100644 index 000000000..2d0e881c0 --- /dev/null +++ b/docs/pep-0798.md @@ -0,0 +1,24 @@ +# PEP 798: Unpacking comprehensions + +Specification: https://peps.python.org/pep-0798/ + +Language version: Python 3.15. Branch dependency: the shared grammar-cache isolation fix. + +## Supported formatting + +List, set, dictionary, and generator unpacking; a generator used as the sole call argument; async comprehensions and conditional generator/mapping heads. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep798_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-0810.md b/docs/pep-0810.md new file mode 100644 index 000000000..d5aa88a97 --- /dev/null +++ b/docs/pep-0810.md @@ -0,0 +1,28 @@ +# PEP 810: Explicit lazy imports + +Specification: https://peps.python.org/pep-0810/ + +Language version: Python 3.15. Branch dependency: the shared grammar-cache isolation fix. + +## Supported formatting + +Both lazy import forms, relative imports, aliases, parenthesized import lists, soft-keyword handling, and import/variable blank-line policy. + +New syntax is parsed by the vendored grammar/tokenizer, independently of the +Python version running YAPF. Executing the formatted program still requires a +compatible interpreter. YAPF is a formatter, not a type checker or a replacement +for compiler-level semantic validation. + +Compiler restrictions on lazy imports (for example, function/class/try scopes, +wildcard imports, and future imports) remain the responsibility of the Python +compiler. Parsing by YAPF is not evidence that a program passes those checks. + +## Regression tests + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.pep810_test +``` + +Tests cover exact output, idempotence, style options, and relevant edge cases. +Native AST/semantic checks run only on an interpreter supporting this PEP; +formatter assertions run on older hosts too. diff --git a/docs/pep-feature-branches.md b/docs/pep-feature-branches.md new file mode 100644 index 000000000..75b2da53f --- /dev/null +++ b/docs/pep-feature-branches.md @@ -0,0 +1,58 @@ +# PEP feature branches + +All changes are based on the uploaded repository at +`12005095296072751e3e4c1f33a047d41b0ce18d`. `main` is unchanged. + +| Branch | Prerequisite | +| --- | --- | +| `fix/grammar-cache-isolation-20261009` | Original `main` | +| `fix/soft-keyword-lookahead-20261009` | Cache isolation | +| `feature/pep-695-type-parameters-20261009` | Cache isolation + lookahead | +| `feature/pep-696-type-defaults-20261009` | PEP 695 | +| `feature/pep-701-fstrings-20261009` | Cache isolation + lookahead | +| `feature/pep-750-template-strings-20261009` | PEP 701 | +| `feature/pep-758-exception-lists-20261009` | Cache isolation + lookahead | +| `feature/pep-798-unpacking-comprehensions-20261009` | Cache isolation + lookahead | +| `feature/pep-810-lazy-imports-20261009` | Cache isolation + lookahead | +| `feature/python-syntax-integration-20261009` | Merges all seven PEP branches | + +Each PEP branch has its own implementation, regression tests, and notes. The +integration branch retains merge ancestry, combines the two contextual keyword +checks in `FormatToken.is_keyword`, adds cross-feature tests, and extends CI/tox +coverage through Python 3.15. It does not squash or replace the feature branches. +For isolated reviews, target PEP 696 at PEP 695, and PEP 750 at PEP 701. + +## Review follow-up + +The original feature commits are retained. Every PEP branch now also includes +`fix/soft-keyword-lookahead-20261009`, which reimplements the shared token buffer +to fix nested speculative parsing and quadratic scanning of old release records. +The new `type` and `lazy` soft keywords exposed those defects: for example, +`match(type)` parsed on original `main` but failed after the PEP additions. +Nine regression test methods cover stream invariants and 68 syntax subcases. +See [the lookahead design note](soft-keyword-lookahead.md). + +No PEP grammar or interpolation scanner was replaced merely for stylistic reasons. +The reviewed integration preserves the original seven implementation branches, +and the follow-up commits are fast-forward descendants of their prior heads. + +The shared cache fix fingerprints the grammar source. Without it, checkouts with +the same YAPF/interpreter version can reuse a different branch's cached grammar. + +## Validation + +Local execution uses CPython 3.13.5 on Linux. Formatter tests run on every host; +native AST comparisons and native semantic tests for newer syntax run only when +the interpreter supports it. Python 3.12/3.14/3.15 downloads were unavailable in +the implementation environment. CI configuration is supplied but was not run on +GitHub; Windows and macOS are not locally validated. + +```bash +PYTHONPATH=third_party:. python -m unittest discover -p '*_test.py' yapftests/ +``` + +This is syntax-formatting support, not type checking or complete compiler +validation. F-string and t-string interiors are preserved, not reformatted. +Long unparenthesized exception lists remain on one physical line. The separate +Unicode-identifier limitation from the earlier audit is not addressed by these +PEP branches. See each PEP's implementation note for scope details. diff --git a/docs/soft-keyword-lookahead.md b/docs/soft-keyword-lookahead.md new file mode 100644 index 000000000..8b47c76c0 --- /dev/null +++ b/docs/soft-keyword-lookahead.md @@ -0,0 +1,29 @@ +# Soft-keyword lookahead isolation + +The PEP 695 and PEP 810 grammars add the contextual keywords `type` and `lazy`. +That exposed two problems in the existing speculative token reader: + +* Nested lookahead could attach tokens to the wrong release record. Valid code + such as `match(type)` and `match (type):\n case _: pass` then failed to parse. +* Release records survived until the end of the file, and every committed token + scanned every earlier record. Ordinary repeated `type(value)` calls therefore + made parsing quadratic rather than linear in the length of the token stream. + +`TokenProxy` now uses one absolute-position buffer for unread tokens and a stack +of independent speculative cursors. Nested routes start immediately after the +parent's currently observed token. Exiting a route restores its parent without +consuming input. Committed tokens are removed from the buffer immediately. +This bounds token storage by outstanding lookahead rather than by the number of +soft keywords already parsed, and makes buffered token access constant-time. +The parser's existing grammar-selection algorithm is unchanged. + +Tests cover replay, competing and nested routes, exceptions, end of input, +negative offsets, prompt removal of consumed tokens, and AST-preserving +formatting of ambiguous calls, subscripts, and match subjects. Buffer lifetime +is asserted structurally rather than with a machine-dependent timing threshold. +The implementation and tests remain compatible with YAPF's Python 3.7 minimum; +native AST comparisons for match statements require Python 3.10 or later. + +```bash +PYTHONPATH=third_party:. python -m unittest yapftests.token_proxy_test +``` diff --git a/third_party/yapf_third_party/_ylib2to3/Grammar.txt b/third_party/yapf_third_party/_ylib2to3/Grammar.txt index bd8a452a3..e1c8d1764 100644 --- a/third_party/yapf_third_party/_ylib2to3/Grammar.txt +++ b/third_party/yapf_third_party/_ylib2to3/Grammar.txt @@ -16,7 +16,7 @@ decorator: '@' namedexpr_test NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite +funcdef: 'def' NAME [typeparams] parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' # The following definition for typedarglist is equivalent to this set of rules: @@ -75,7 +75,7 @@ vfplist: vfpdef (',' vfpdef)* [','] stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) + import_stmt | global_stmt | exec_stmt | assert_stmt | type_stmt) expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) annassign: ':' test ['=' (yield_expr|testlist_star_expr)] @@ -93,7 +93,9 @@ continue_stmt: 'continue' return_stmt: 'return' [testlist_star_expr] yield_stmt: yield_expr raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] -import_stmt: import_name | import_from +# Factor the soft keyword so the two import forms remain LL(1). +import_stmt: import_name | import_from | lazy_import +lazy_import: "lazy" (import_name | import_from) import_name: 'import' dotted_as_names import_from: ('from' ('.'* dotted_name | '.'+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) @@ -119,7 +121,8 @@ try_stmt: ('try' ':' suite with_stmt: 'with' asexpr_test (',' asexpr_test)* ':' suite # NB compile.c makes sure that the default except clause is last -except_clause: 'except' ['*'] [test [(',' | 'as') test]] +# PEP 758: a comma-separated exception list cannot have an 'as' target. +except_clause: 'except' ['*'] [test ['as' test | ',' [test (',' test)* [',']]]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT # Backward compatibility cruft to support: @@ -161,7 +164,9 @@ atom: ('(' [yield_expr|testlist_gexp] ')' | '`' testlist1 '`' | NAME | NUMBER | STRING+ | '.' '.' '.') listmaker: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) -testlist_gexp: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) +# PEP 798 permits full expressions in an unpacking generator head. +star_gexp: '*' test +testlist_gexp: (namedexpr_test|star_gexp) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: (subscript|star_expr) (',' (subscript|star_expr))* [','] @@ -169,12 +174,18 @@ subscript: test [':=' test] | [test] ':' [test] [sliceop] sliceop: ':' [test] exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] testlist: test (',' test)* [','] -dictsetmaker: ( ((test ':' asexpr_test | '**' expr) +dictsetmaker: ( ((test ':' asexpr_test | '**' test) (comp_for | (',' (test ':' asexpr_test | '**' expr))* [','])) | ((test [':=' test] | star_expr) (comp_for | (',' (test [':=' test] | star_expr))* [','])) ) -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite +type_stmt: "type" NAME [typeparams] '=' test +typeparams: '[' typeparamslist ']' +typeparamslist: typeparam (',' typeparam)* [','] +# PEP 696: only a TypeVarTuple default may be a starred expression. +typeparam: NAME [':' test] ['=' test] | '*' NAME ['=' (test|star_expr)] | '**' NAME ['=' test] + +classdef: 'class' NAME [typeparams] ['(' [arglist] ')'] ':' suite arglist: argument (',' argument)* [','] @@ -190,7 +201,7 @@ argument: ( test [comp_for] | test 'as' test | test '=' asexpr_test | '**' test | - '*' test ) + '*' test [comp_for] ) comp_iter: comp_for | comp_if comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py b/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py index 76b31a11c..dc32a1946 100644 --- a/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py @@ -14,6 +14,7 @@ __all__ = ['Driver', 'load_grammar'] +import hashlib import io import logging import os @@ -22,12 +23,11 @@ # Python imports from contextlib import contextmanager from dataclasses import dataclass -from dataclasses import field from pathlib import Path from typing import Any +from typing import Dict from typing import Iterator from typing import List -from typing import Optional from platformdirs import user_cache_dir @@ -42,72 +42,72 @@ @dataclass -class ReleaseRange: - start: int - end: Optional[int] = None - tokens: List[Any] = field(default_factory=list) +class _Lookahead: + """An independent cursor for one speculative parser invocation.""" - def lock(self) -> None: - total_eaten = len(self.tokens) - self.end = self.start + total_eaten + start: int + next_position: int class TokenProxy: + """Buffer unread tokens once, independently of speculative parser cursors. + + A release scope observes tokens without consuming the main iterator. Nested + scopes start after the token most recently observed by their parent, and + returning from a nested scope restores the parent's cursor. All scopes share + the same indexed buffer, so competing parser routes see identical tokens. + Committed tokens are discarded immediately rather than retaining and scanning + one release record for every soft keyword in the input. + """ def __init__(self, generator: Any) -> None: - self._tokens = generator - self._counter = 0 - self._release_ranges: List[ReleaseRange] = [] + self._tokens = iter(generator) + self._position = 0 + self._read_position = 0 + self._buffer: Dict[int, Any] = {} + self._lookahead: List[_Lookahead] = [] + + def _get(self, position: int) -> Any: + while self._read_position <= position: + self._buffer[self._read_position] = next(self._tokens) + self._read_position += 1 + return self._buffer[position] @contextmanager def release(self) -> Iterator['TokenProxy']: - release_range = ReleaseRange(self._counter) - self._release_ranges.append(release_range) + start = ( + self._lookahead[-1].next_position + if self._lookahead else self._position) + self._lookahead.append(_Lookahead(start, start)) try: yield self finally: - # Lock the last release range to the final position that - # has been eaten. - release_range.lock() + self._lookahead.pop() def eat(self, point: int) -> Any: - eaten_tokens = self._release_ranges[-1].tokens - if point < len(eaten_tokens): - return eaten_tokens[point] - else: - while point >= len(eaten_tokens): - token = next(self._tokens) - eaten_tokens.append(token) - return token + if point < 0: + raise IndexError('lookahead offset must be nonnegative') + scope = self._lookahead[-1] + position = scope.start + point + value = self._get(position) + scope.next_position = position + 1 + return value def __iter__(self) -> 'TokenProxy': return self def __next__(self) -> Any: - # If the current position is already compromised (looked up) - # return the eaten token, if not just go further on the given - # token producer. - for release_range in self._release_ranges: - assert release_range.end is not None - - start, end = release_range.start, release_range.end - if start <= self._counter < end: - token = release_range.tokens[self._counter - start] - break - else: - token = next(self._tokens) - self._counter += 1 - return token + value = self._get(self._position) + del self._buffer[self._position] + self._position += 1 + return value def can_advance(self, to: int) -> bool: - # Try to eat, fail if it can't. The eat operation is cached - # so there wont be any additional cost of eating here try: self.eat(to) except StopIteration: return False - else: - return True + return True class Driver(object): @@ -187,8 +187,8 @@ def parse_string(self, text, debug=False): return self.parse_tokens(tokens, debug) -def _generate_pickle_name(gt): - # type:(str) -> str +def _generate_pickle_name(gt, source=None): + # type:(str, bytes | None) -> str """Get the filepath to write a pickle file to given the path of a grammar textfile. @@ -196,6 +196,7 @@ def _generate_pickle_name(gt): Args: gt (str): path to grammar text file + source (bytes): optional source snapshot used to isolate cache entries Returns: str: path to pickle file @@ -205,6 +206,8 @@ def _generate_pickle_name(gt): head, tail = os.path.splitext(grammar_textfile_name) if tail == '.txt': tail = '' + if source is not None: + head += '-' + hashlib.sha256(source).hexdigest() cache_dir = user_cache_dir( appname='YAPF', appauthor='Google', version=yapf_version) return cache_dir + os.sep + head + tail + '-py' + '.'.join( @@ -220,20 +223,24 @@ def load_grammar(gt='Grammar.txt', """Load the grammar (maybe from a pickle).""" if logger is None: logger = logging.getLogger() - gp = _generate_pickle_name(gt) if gp is None else gp - grammar_text = gt + # Different checkouts can have the same package/interpreter version and + # file timestamps, but different grammars. Key the cache by the source, + # and parse the same snapshot so no checkout can reuse another's grammar. + try: + with open(gt, 'rb') as source_file: + source = source_file.read() + except FileNotFoundError: + gt_basename = os.path.basename(gt) + source = pkgutil.get_data('yapf_third_party._ylib2to3', gt_basename) + if source is None: + raise RuntimeError('Failed to load grammar %s from package' % gt_basename) + gp = _generate_pickle_name(gt, source) if gp is None else gp + grammar_text = io.StringIO(source.decode(encoding='utf-8')) try: newer = _newer(gp, gt) except OSError as err: logger.debug('OSError, could not check if newer: %s', err.args) newer = True - if not os.path.exists(gt): - # Assume package data - gt_basename = os.path.basename(gt) - pd = pkgutil.get_data('yapf_third_party._ylib2to3', gt_basename) - if pd is None: - raise RuntimeError('Failed to load grammer %s from package' % gt_basename) - grammar_text = io.StringIO(pd.decode(encoding='utf-8')) if force or not newer: g = pgen.generate_grammar(grammar_text) if save: diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/fstring.py b/third_party/yapf_third_party/_ylib2to3/pgen2/fstring.py new file mode 100644 index 000000000..73f03e090 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/fstring.py @@ -0,0 +1,207 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Read a complete f-string or t-string without changing its source. + +PEP 701 permits the enclosing quote, comments and newlines in replacement +fields. A regular expression for an ordinary string cannot find the end of +such an f-string. This scanner follows the lexical contexts instead, retaining +YAPF's representation of an entire string literal as a single STRING token. +PEP 750 template strings use the same lexical contexts, including nested +f-strings and t-strings. Replacement text is preserved because it is +observable in template Interpolation.expression and debug expressions. +This scanner deliberately does not parse or reformat those expressions. +""" + +import re + +# Put triple quotes before single quotes, and two-letter prefixes before +# one-letter prefixes. Identifiers are consumed separately in expressions so +# that a suffix of an identifier cannot be mistaken for a string prefix. +STRING_START = re.compile( + r'''(?i:(fr|rf|tr|rt|br|rb|r|u|b|f|t)?)("""|\'\'\'|"|\')''') +FSTRING_START = r'''(?:[fF][rR]?|[rR][fF])(?:"""|\'\'\'|"|\')''' +TSTRING_START = FSTRING_START.replace('[fF]', '[tT]') +INTERPOLATED_STRING_START = '(?:' + FSTRING_START + '|' + TSTRING_START + ')' +_NAME = re.compile(r'\w+') +_CLOSING = {'(': ')', '[': ']', '{': '}'} + + +class FStringError(Exception): + """A lexical error, with the same arguments as tokenize.TokenError.""" + + +def scan_interpolated_string(readline, line, lineno, column): + """Return (value, end_position, last_line, physical_lines) for a literal. + + ``line`` is the physical line already read by the caller. Only subsequent + physical lines belonging to this literal are read from ``readline``. The + caller resumes tokenization at ``end_position`` in ``last_line``. + """ + scanner = _Scanner(readline, line, lineno, column) + scanner.scan_string() + physical_lines = ''.join(scanner.lines) + if len(scanner.lines) == 1: + value = line[column:scanner.column] + else: + value = ( + line[column:] + ''.join(scanner.lines[1:-1]) + + scanner.line[:scanner.column]) + return value, (scanner.lineno, scanner.column), scanner.line, physical_lines + + +# Retain the PEP 701 entry point for existing scanner clients. +scan_fstring = scan_interpolated_string + + +class _Scanner: + + def __init__(self, readline, line, lineno, column): + self.readline = readline + self.line = line + self.lineno = lineno + self.column = column + self.start = (lineno, column) + self.lines = [line] + prefix = STRING_START.match(line, column) + self.template = bool(prefix and 't' in (prefix.group(1) or '').lower()) + + def error(self, message): + if self.template: + message = message.replace('f-string', 't-string') + raise FStringError(message, self.start) + + def peek(self): + while self.column >= len(self.line): + try: + line = self.readline() + except StopIteration: + line = '' + if not line: + self.error('EOF in multi-line f-string') + self.line = line + self.lines.append(line) + self.lineno += 1 + self.column = 0 + return self.line[self.column] + + def scan_string(self): + match = STRING_START.match(self.line, self.column) + if match is None: + self.error('expected string literal') + prefix, quote = match.groups() + prefix = (prefix or '').lower() + formatted = 'f' in prefix or 't' in prefix + raw = 'r' in prefix + self.column = match.end() + while True: + char = self.peek() + if self.line.startswith(quote, self.column): + self.column += len(quote) + return + if char in '\r\n' and len(quote) == 1: + self.error('unterminated string literal') + if char == '\\': + self.scan_escape(raw, formatted) + elif formatted and char == '{': + if self.line.startswith('{{', self.column): + self.column += 2 + else: + self.column += 1 + self.scan_replacement(quote, raw) + elif formatted and char == '}': + if not self.line.startswith('}}', self.column): + self.error("f-string: single '}' is not allowed") + self.column += 2 + else: + self.column += 1 + + def scan_escape(self, raw, formatted): + self.column += 1 + char = self.peek() + if formatted and char in '{}': + # A backslash does not escape an f-string replacement-field delimiter. + return + if (formatted and not raw and char == 'N' and + self.line.startswith('N{', self.column)): + # Braces in a named Unicode escape are not replacement fields. + self.column += 2 + while self.peek() != '}': + if self.peek() in '\r\n': + self.error('unterminated Unicode name escape') + self.column += 1 + self.column += 1 + elif self.line.startswith('\r\n', self.column): + self.column += 2 + else: + self.column += 1 + + def scan_replacement(self, quote, raw): + brackets = [] + while True: + char = self.peek() + if STRING_START.match(self.line, self.column): + self.scan_string() + elif char == '#': + # Quotes and braces in comments have no lexical significance. + while self.peek() not in '\r\n': + self.column += 1 + elif char == '\\': + self.column += 1 + char = self.peek() + if self.line.startswith('\r\n', self.column): + self.column += 2 + elif char == '\n': + self.column += 1 + else: + self.error('unexpected character after line continuation character') + elif char in _CLOSING: + brackets.append(_CLOSING[char]) + self.column += 1 + elif char in ')]}': + if brackets: + if brackets.pop() != char: + self.error('f-string: mismatched bracket in replacement field') + self.column += 1 + elif char == '}': + self.column += 1 + return + else: + self.error('f-string: unmatched closing bracket') + elif not brackets and char == ':': + self.column += 1 + self.scan_format_spec(quote, raw) + return + else: + name = _NAME.match(self.line, self.column) + self.column = name.end() if name else self.column + 1 + + def scan_format_spec(self, quote, raw): + while True: + char = self.peek() + if self.line.startswith(quote, self.column): + self.error("f-string: expecting '}'") + if char in '\r\n' and len(quote) == 1: + self.error('f-string: newline in format specifier') + if char == '}': + self.column += 1 + return + if char == '{': + # Unlike literal text, format specs do not escape doubled braces. + # For example, {{}} here is a replacement field containing a dict. + self.column += 1 + self.scan_replacement(quote, raw) + elif char == '\\': + self.scan_escape(raw, True) + else: + self.column += 1 diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py b/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py index dda83296f..3548defc8 100644 --- a/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py @@ -33,6 +33,7 @@ from codecs import BOM_UTF8 from codecs import lookup +from . import fstring from . import token from .token import ASYNC from .token import AWAIT @@ -127,9 +128,11 @@ def _combinations(*l): # noqa: E741 ContStr = group( _litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n', Comment, fstring.INTERPOLATED_STRING_START, + Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) +interpolated_string_prog = re.compile(fstring.INTERPOLATED_STRING_START) tokenprog, pseudoprog, single3prog, double3prog = map( re.compile, (Token, PseudoToken, Single3, Double3)) @@ -517,6 +520,18 @@ def generate_tokens(readline): yield stashed stashed = None yield (COMMENT, token, spos, epos, line) + elif interpolated_string_prog.match(token): + if stashed: + yield stashed + stashed = None + try: + value, epos, line, physical_lines = fstring.scan_interpolated_string( + readline, line, lnum, start) + except fstring.FStringError as error: + raise TokenError(*error.args) from error + lnum, pos = epos + max = len(line) + yield (STRING, value, spos, epos, physical_lines) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) diff --git a/tox.ini b/tox.ini index 5e5b11e79..350d89b74 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ requires = tox<4 tox-pyenv tox-wheel -envlist = py37,py38,py39,py310,py311,py312 +envlist = py37,py38,py39,py310,py311,py312,py313,py314,py315 # tox-wheel alias for `wheel_pep517 = true` isolated_build = True distshare = ./dist diff --git a/yapf/pytree/pytree_unwrapper.py b/yapf/pytree/pytree_unwrapper.py index 80e050fbd..5552c1e6d 100644 --- a/yapf/pytree/pytree_unwrapper.py +++ b/yapf/pytree/pytree_unwrapper.py @@ -299,6 +299,10 @@ def Visit_arglist(self, node): # pylint: disable=invalid-name _DetermineMustSplitAnnotation(node) self.DefaultNodeVisit(node) + def Visit_typeparamslist(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + def Visit_typedargslist(self, node): # pylint: disable=invalid-name _DetermineMustSplitAnnotation(node) self.DefaultNodeVisit(node) @@ -365,8 +369,11 @@ def _IdentifyParameterLists(line): for tok in line.tokens: # Identify parameter list objects. if subtypes.FUNC_DEF in tok.subtypes: - assert tok.next_token.value == '(' - func_stack.append(tok.next_token) + opening = tok.next_token + if opening.value == '[': + opening = opening.matching_bracket.next_token + assert opening.value == '(' + func_stack.append(opening) continue if func_stack and tok.value == ')': diff --git a/yapf/pytree/split_penalty.py b/yapf/pytree/split_penalty.py index 03b36384a..42660ed02 100644 --- a/yapf/pytree/split_penalty.py +++ b/yapf/pytree/split_penalty.py @@ -81,13 +81,13 @@ def Visit_import_as_names(self, node): # pyline: disable=invalid-name prev_child = child def Visit_classdef(self, node): # pylint: disable=invalid-name - # classdef ::= 'class' NAME ['(' [arglist] ')'] ':' suite + # classdef ::= 'class' NAME [typeparams] ['(' [arglist] ')'] ':' suite # # NAME _SetUnbreakable(node.children[1]) - if len(node.children) > 4: - # opening '(' - _SetUnbreakable(node.children[2]) + for child in node.children[2:-2]: + if child.type == grammar_token.LPAR: + _SetUnbreakable(child) # ':' _SetUnbreakable(node.children[-2]) self.DefaultNodeVisit(node) @@ -133,6 +133,19 @@ def Visit_lambdef(self, node): # pylint: disable=invalid-name else: _SetExpressionPenalty(node, VERY_STRONGLY_CONNECTED) + def Visit_typeparams(self, node): # pylint: disable=invalid-name + self.Visit_parameters(node) + + def Visit_typeparam(self, node): # pylint: disable=invalid-name + self.Visit_tname(node) + for index, child in enumerate(node.children[:-1]): + if child.type == grammar_token.EQUAL: + _SetSplitPenalty(child, NAMED_ASSIGN) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) + if node.children[0].type in (grammar_token.STAR, grammar_token.DOUBLESTAR): + _SetUnbreakable(node.children[1]) + def Visit_parameters(self, node): # pylint: disable=invalid-name # parameters ::= '(' [typedargslist] ')' self.DefaultNodeVisit(node) @@ -425,6 +438,9 @@ def Visit_comparison(self, node): # pylint: disable=invalid-name else: _IncreasePenalty(node, COMPARISON) + def Visit_star_gexp(self, node): # pylint: disable=invalid-name + self.Visit_star_expr(node) + def Visit_star_expr(self, node): # pylint: disable=invalid-name # star_expr ::= '*' expr self.DefaultNodeVisit(node) diff --git a/yapf/pytree/subtype_assigner.py b/yapf/pytree/subtype_assigner.py index e3b32777a..7589b367e 100644 --- a/yapf/pytree/subtype_assigner.py +++ b/yapf/pytree/subtype_assigner.py @@ -107,6 +107,22 @@ def markAsDictSetGenerator(node): elif last_was_colon: unpacking = False + def Visit_type_stmt(self, node): # pylint: disable=invalid-name + # An alias assignment is not a default/named argument assignment. + self.Visit_expr_stmt(node) + + def Visit_typeparam(self, node): # pylint: disable=invalid-name + # Type defaults use assignment spacing, not value-parameter/keyword-arg + # spacing. Only direct children are annotated: a call in a default must + # retain its own keyword-argument spacing. + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf): + subtype = ( + subtypes.ASSIGN_OPERATOR if child.value == '=' else + _ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, subtypes.NONE)) + _AppendTokenSubtype(child, subtype) + def Visit_expr_stmt(self, node): # pylint: disable=invalid-name # expr_stmt ::= testlist_star_expr (augassign (yield_expr|testlist) # | ('=' (yield_expr|testlist_star_expr))*) @@ -148,6 +164,9 @@ def Visit_comparison(self, node): # pylint: disable=invalid-name for grandchild in child.children: _AppendTokenSubtype(grandchild, subtypes.BINARY_OPERATOR) + def Visit_star_gexp(self, node): # pylint: disable=invalid-name + self.Visit_star_expr(node) + def Visit_star_expr(self, node): # pylint: disable=invalid-name # star_expr ::= '*' expr for child in node.children: diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 06f3455d9..31e4f69b0 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -1183,6 +1183,8 @@ def _LastTokenInLine(current): def _IsFunctionDefinition(current): prev = current.previous_token + if prev and prev.value == ']' and prev.matching_bracket: + prev = prev.matching_bracket.previous_token return current.value == '(' and prev and subtypes.FUNC_DEF in prev.subtypes diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py index 141c2650e..5f8faf69a 100644 --- a/yapf/yapflib/format_token.py +++ b/yapf/yapflib/format_token.py @@ -280,6 +280,14 @@ def is_continuation(self): @property @lru_cache() def is_keyword(self): + if (self.value == 'type' and self.node.parent is not None and + type_repr(self.node.parent.type) == 'type_stmt' and + self.node is self.node.parent.children[0]): + return True + if (self.value == 'lazy' and self.node.parent is not None and + type_repr(self.node.parent.type) == 'lazy_import' and + self.node is self.node.parent.children[0]): + return True return keyword.iskeyword( self.value) or (self.value == 'match' and type_repr(self.node.parent.type) == 'match_stmt') or ( @@ -300,13 +308,13 @@ def is_string(self): @property def is_multiline_string(self): - """Test if this string is a multiline string. + """Recognize triple-quoted strings and strings with physical newlines. - Returns: - A multiline string always ends with triple quotes, so if it is a string - token, inspect the last 3 characters and return True if it is a triple - double or triple single quote mark. + PEP 701 permits newlines inside a single-quoted f-string field. + Preserve the existing treatment of one-line triple-quoted strings. """ + if self.is_string and '\n' in self.value: + return True return self.is_string and self.value.endswith(('"""', "'''")) @property diff --git a/yapf/yapflib/logical_line.py b/yapf/yapflib/logical_line.py index 4433276f7..4c030cc1c 100644 --- a/yapf/yapflib/logical_line.py +++ b/yapf/yapflib/logical_line.py @@ -296,9 +296,16 @@ def _SpaceRequiredBetween(left, right, is_line_disabled): if _IsIdNumberStringToken(left) and _IsIdNumberStringToken(right): # Spaces between keyword, string, number, and identifier tokens. return True - if lval == ',' and rval == ':': - # We do want a space between a comma and colon. + if lval == 'except' and rval == '*': + # The exception-group marker is attached to the except keyword. + return False + if lval == '*' and left.previous_token and left.previous_token.value == 'except': return True + if lval == ',' and rval == ':': + # A PEP 758 exception list may end in a comma. Unlike a multidimensional + # slice, the following colon terminates the clause, not another element. + return not (left.node.parent is not None and + left.node.parent.type == python_symbols.except_clause) if style.Get('SPACE_INSIDE_BRACKETS'): # Supersede the "no space before a colon or comma" check. if left.OpensScope() and rval == ':': diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index ff0952543..fd69b21a9 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -643,7 +643,8 @@ def _CalculateNumberOfNewlines(first_token, indent_depth, prev_line, return NO_BLANK_LINES if first_token.is_name and not indent_depth: - if prev_line.first.value in {'from', 'import'}: + if (prev_line.first.value in {'from', 'import'} or + (prev_line.first.value == 'lazy' and prev_line.first.is_keyword)): # Support custom number of blank lines between top-level imports and # variable definitions. return 1 + style.Get( diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py index aa010bb8f..3e3308293 100644 --- a/yapf/yapflib/yapf_api.py +++ b/yapf/yapflib/yapf_api.py @@ -282,8 +282,12 @@ def _MarkLinesToFormat(llines, lines): """Skip sections of code that we shouldn't reformat.""" if lines: for uwline in llines: + last_lineno = uwline.last.lineno + if uwline.last.is_multiline_string: + # A selection inside an opaque string token still selects its statement. + last_lineno += uwline.last.value.count('\n') uwline.disable = not lines.intersection( - range(uwline.lineno, uwline.last.lineno + 1)) + range(uwline.lineno, last_lineno + 1)) # Now go through the lines and disable any lines explicitly marked as # disabled. diff --git a/yapftests/fstring_scanner_test.py b/yapftests/fstring_scanner_test.py new file mode 100644 index 000000000..efa92fc69 --- /dev/null +++ b/yapftests/fstring_scanner_test.py @@ -0,0 +1,176 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Lossless scanner tests, independent of YAPF's formatter implementation.""" + +import ast +import io +import itertools +import random +import sys +import unittest +import warnings + +from yapf_third_party._ylib2to3.pgen2 import fstring + +_CASES = [ + 'f"{data["key"]}"', + "f'{data['key']}'", + 'f"{f"{f"{f"{f"{1 + 1}"}"}"}"}"', + 'f"{x # outer quote \" and } are ignored\n}"', + 'f"{x\n}"', + 'f"{(x\n + y)}"', + 'f"{\"\\n\".join(items)}"', + 'f"{x + \\\n y}"', + 'f"{(lambda x: x + 1)(2)}"', + 'f"{ {\"k\": \"v\"}[\"k\"] }"', + 'f"{[x for x in values]}"', + 'f"{(value := 2)}"', + 'f"{value!r:>{width}}"', + 'f"{value = !r:>{width}}"', + 'f"{value:{{}}}"', + 'f"{value:}}}"', + 'f"{value:\\N{LEFT CURLY BRACKET}}"', + 'f"\\N{LEFT CURLY BRACKET} {value}"', + 'fr"\\N{value}"', + 'f"\\{value}"', + 'f"{{escaped}} {value}"', + 'f"\\\\{value}"', + 'f"a\\\nb{value}"', + 'f"{\"\"\"inside\ntriple\"\"\"}"', + 'f"""{value:\n>10}"""', + 'f"""{data["""key"""]}"""', + 'f"{value:{width # ignored \" }\n}}"', + 'f"{value:{f"{width}"}}"', + 'f"{await foo()}"', + 'f"{(yield value)}"', + 'f"{(x async for x in y)}"', + 'f"{ {x:y for x,y in z} }"', + 'f"{data[1:2]}"', + 'f"{value != other}"', +] +for prefix, quote, body in itertools.product( + ['f', 'F', 'fr', 'fR', 'Fr', 'FR', 'rf', 'rF', 'Rf', 'RF'], + ['"', "'", '"""', "'''"], [ + '', 'text', '{{}}', '{{{value}}}', '{value!r}', '{value=}', + '{value:>{width}}', '{ {1:2} }', '{value:{width:{precision}}}' + ]): + _CASES.append(prefix + quote + body + quote) + + +class FStringScannerTest(unittest.TestCase): + + def checkLiteral(self, literal, newline='\n'): + if sys.version_info >= (3, 12): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + ast.parse(literal) + source = 'value = ' + literal + '; after = 1' + newline + reader = io.StringIO(source).readline + first = reader() + value, end, last_line, physical = fstring.scan_fstring(reader, first, 1, 8) + self.assertEqual(literal, value) + column = (8 + len(literal) if '\n' not in literal else len( + literal.rsplit('\n', 1)[1])) + self.assertEqual((literal.count('\n') + 1, column), end) + self.assertEqual('; after = 1' + newline, last_line[end[1]:]) + self.assertEqual(source, physical) + self.assertEqual('', reader()) + + def testFixedAndPrefixMatrix(self): + for literal in _CASES: + with self.subTest(literal=literal): + self.checkLiteral(literal) + + def testCrLf(self): + for literal in _CASES: + if '\n' in literal: + with self.subTest(literal=literal): + self.checkLiteral(literal.replace('\n', '\r\n'), '\r\n') + + def testDoesNotReadAhead(self): + literal = 'f"{data["key"]}"' + + def unexpectedRead(): + self.fail('readline called after the literal already ended') + + value, end, last, physical = fstring.scan_fstring(unexpectedRead, literal, + 1, 0) + self.assertEqual(literal, value) + self.assertEqual((1, len(literal)), end) + self.assertEqual(literal, last) + self.assertEqual(literal, physical) + + def testStopIterationReader(self): + literal = 'f"{value # comment\n}"' + lines = iter(literal.splitlines(keepends=True)) + first = next(lines) + value, _, _, _ = fstring.scan_fstring(lines.__next__, first, 5, 0) + self.assertEqual(literal, value) + with self.assertRaises(fstring.FStringError): + fstring.scan_fstring(iter(()).__next__, 'f"{value', 1, 0) + + def testMalformedLexicalContexts(self): + literals = [ + 'f"{x', 'f"abc', 'f"}"', 'f"{(x]}"', 'f"{x # no newline', 'f"{x:\n}"', + 'f"line\nbreak"', 'f"\\N{no closing brace', 'f"{x:\">10}"' + ] + for literal in literals: + with self.subTest(literal=literal): + with self.assertRaises(fstring.FStringError): + fstring.scan_fstring(io.StringIO('').readline, literal, 1, 0) + + @unittest.skipUnless(sys.version_info >= (3, 12), 'requires a PEP 701 oracle') + def testGeneratedNativeValidLiterals(self): + rng = random.Random(701) + expressions = [ + 'value', + 'data["key"]', + "data['key']", + '"\\n".join(items)', + '(lambda x: x + 1)(2)', + '(a := value)', + '[x for x in y]', + '{"key": "value"}', + 'value\n', + 'value # comment } "\n', + '\"\"\"embedded\nstring\"\"\"', + 'f"{value}"', + 'f"{value["key"]}"', + 'value + \\\nother', + 'data[1:2]', + ] + valid = 0 + for _ in range(3000): + prefix = rng.choice(['f', 'F', 'rf', 'FR']) + quote = rng.choice(['"', "'", '\"\"\"', "\'\'\'"]) + parts = [] + for _ in range(rng.randrange(1, 6)): + parts.append(rng.choice(['text', '{{escaped}}', ''])) + expr = rng.choice(expressions) + conversion = rng.choice(['', '!r', '!s', '!a']) + fmt = rng.choice(['', ':>10', ':>{width}', ':{width}.{precision}f']) + parts.append('{' + expr + conversion + fmt + '}') + literal = prefix + quote + ''.join(parts) + quote + try: + ast.parse(literal) + except SyntaxError: + continue + with self.subTest(literal=literal): + self.checkLiteral(literal) + valid += 1 + self.assertGreater(valid, 2000) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/grammar_cache_test.py b/yapftests/grammar_cache_test.py new file mode 100644 index 000000000..26a45b61c --- /dev/null +++ b/yapftests/grammar_cache_test.py @@ -0,0 +1,84 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Grammar cache entries must follow the contents, not checkout timestamps.""" + +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from yapf.pytree import pytree_utils + +# Initialize YAPF before accessing its vendored driver (the driver imports YAPF). +driver = pytree_utils.driver + + +class GrammarCacheTest(unittest.TestCase): + + def testDifferentCheckoutsWithSameBasenameAndTimestamp(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / 'cache' + with mock.patch.object(driver, 'user_cache_dir', return_value=str(cache)): + grammars = [] + for i, keyword in enumerate(('first', 'second', 'first')): + source = root / str(i) / 'Grammar.txt' + source.parent.mkdir() + source.write_text("start: '%s' NEWLINE ENDMARKER\n" % keyword) + os.utime(str(source), (1, 1)) + grammars.append(driver.load_grammar(str(source))) + self.assertIn('first', grammars[0].keywords) + self.assertIn('second', grammars[1].keywords) + self.assertNotIn('first', grammars[1].keywords) + self.assertIn('first', grammars[2].keywords) + self.assertEqual(2, len(list(cache.glob('*.pickle')))) + + def testSourceChangesWithPreservedTimestamp(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'Grammar.txt' + with mock.patch.object(driver, 'user_cache_dir', return_value=str(root)): + for keyword in ('before', 'after'): + source.write_text("start: '%s' NEWLINE ENDMARKER\n" % keyword) + os.utime(str(source), (1, 1)) + grammar = driver.load_grammar(str(source)) + self.assertIn(keyword, grammar.keywords) + + def testPackageSourceIsAlsoFingerprintIsolated(self): + with tempfile.TemporaryDirectory() as directory: + with mock.patch.object(driver, 'user_cache_dir', return_value=directory): + for keyword in ('first', 'second'): + source = ("start: '%s' NEWLINE ENDMARKER\n" % keyword).encode() + with mock.patch.object( + driver.pkgutil, 'get_data', return_value=source): + grammar = driver.load_grammar( + os.path.join(directory, 'missing.txt')) + self.assertIn(keyword, grammar.keywords) + + def testExplicitCacheAndForceStillWork(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'Grammar.txt' + target = root / 'explicit.pickle' + source.write_text("start: 'original' NEWLINE ENDMARKER\n") + driver.load_grammar(str(source), gp=str(target)) + self.assertTrue(target.is_file()) + source.write_text("start: 'changed' NEWLINE ENDMARKER\n") + grammar = driver.load_grammar(str(source), gp=str(target), force=True) + self.assertIn('changed', grammar.keywords) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep695_test.py b/yapftests/pep695_test.py new file mode 100644 index 000000000..94ceda7d8 --- /dev/null +++ b/yapftests/pep695_test.py @@ -0,0 +1,164 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PEP 695 regression tests; new syntax stays in string fixtures. + +Native AST comparisons run on Python 3.12 and newer. +""" + +import ast +import sys +import unittest + +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token + +from yapf.yapflib import format_token +from yapf.yapflib.yapf_api import FormatCode + + +class Pep695Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 12): + self.assertEqual( + ast.dump(ast.parse(source, feature_version=(3, 12))), + ast.dump(ast.parse(result, feature_version=(3, 12)))) + return result + + def testTypeAliases(self): + cases = [ + ('type Alias=int\n', 'type Alias = int\n'), + ('type Vector[T]=list[T]\n', 'type Vector[T] = list[T]\n'), + ('type Pair[T,U]=tuple[T,U]\n', 'type Pair[T, U] = tuple[T, U]\n'), + ('type Bounded[T:int]=list[T]\n', 'type Bounded[T: int] = list[T]\n'), + ('type Constrained[T:(str,bytes)]=list[T]\n', + 'type Constrained[T: (str, bytes)] = list[T]\n'), + ('type Shape[* Ts]=tuple[* Ts]\n', 'type Shape[*Ts] = tuple[*Ts]\n'), + ('type Callback[** P]=Callable[P,int]\n', + 'type Callback[**P] = Callable[P, int]\n'), + ('type type[T]=tuple[T,type]\n', 'type type[T] = tuple[T, type]\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testGenericDefinitions(self): + cases = [ + ('def identity[T](value:T)->T:\n return value\n', + 'def identity[T](value: T) -> T:\n return value\n'), + ('async def identity[T](value:T)->T:\n return value\n', + 'async def identity[T](value: T) -> T:\n return value\n'), + ('class Box[T]:\n pass\n', 'class Box[T]:\n pass\n'), + ('class Box[T](Base[T]):\n pass\n', + 'class Box[T](Base[T]):\n pass\n'), + ('def pack[T,* Ts,** P](value:T,*args: *Ts,**kwargs:P.kwargs):\n' + ' return value\n', + 'def pack[T, *Ts, **P](value: T, *args: *Ts, **kwargs: P.kwargs):\n' + ' return value\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testSoftKeywordRemainsAnIdentifier(self): + for source, expected in [ + ('type=object\n', 'type = object\n'), + ('value=type(item)\n', 'value = type(item)\n'), + ('type[T]=value\n', 'type[T] = value\n'), + ('def type(value):\n return value\n', + 'def type(value):\n return value\n'), + ('type(type,type=type)\n', 'type(type, type=type)\n'), + ]: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testDetachedTypeNameIsNotAKeyword(self): + leaf = pytree.Leaf(token.NAME, 'type') + wrapped = format_token.FormatToken(leaf, 'NAME') + self.assertFalse(wrapped.is_keyword) + self.assertTrue(wrapped.is_name) + + def testWrappedTypeParameters(self): + sources = [ + 'type Mapping[LongKeyParameter, LongValueParameter] = ' + 'dict[LongKeyParameter, LongValueParameter]\n', + 'def convert[LongInputParameter, LongOutputParameter](' + 'value: LongInputParameter, fallback: LongOutputParameter = None' + ') -> LongOutputParameter:\n return fallback\n', + 'class Collection[LongKeyParameter, LongValueParameter](' + 'Base[LongKeyParameter, LongValueParameter]):\n pass\n', + 'type Alias[\n T, # key\n U, # value\n] = tuple[T, U]\n', + 'def identity[\n T, # retain this comment\n](value: T) -> T:\n' + ' return value\n', + '@decorate\nasync def identity[T](value: T) -> T:\n' + ' return value\n', + 'def outer[T](value: T):\n' + ' def inner[U](other: U) -> U:\n' + ' return other\n' + ' return inner(value)\n', + ] + for name in ['pep8', 'google', 'yapf', 'facebook']: + for source in sources: + with self.subTest(style=name, source=source): + result = self.assertFormatting( + source, + style={ + 'BASED_ON_STYLE': name, + 'COLUMN_LIMIT': 48, + 'DEDENT_CLOSING_BRACKETS': True, + }) + for line in source.splitlines(): + if '#' in line: + self.assertIn(line[line.index('#'):], result) + + def testTrailingTypeParameterCommas(self): + cases = [ + ('type Alias[T,U,]=tuple[T,U]\n', + 'type Alias[\n T,\n U,\n] = tuple[T, U]\n'), + ('def identity[T,U,](x:T,y:U)->tuple[T,U]:\n return x,y\n', + 'def identity[\n T,\n U,\n]' + '(x: T, y: U) -> tuple[T, U]:\n return x, y\n'), + ('class Container[T,* Ts,** P,](Base):\n pass\n', + 'class Container[\n T,\n *Ts,\n **P,\n](Base):\n pass\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting( + source, + expected, + style={ + 'BASED_ON_STYLE': 'pep8', + 'DEDENT_CLOSING_BRACKETS': True + }) + + def testTypeParameterComments(self): + for source in [ + 'type Alias[\n # leading\n T # trailing\n]=list[T]\n', + 'def f[\n # leading\n T # trailing\n](value:T):\n return value\n', + 'class Box[\n # leading\n T # trailing\n]:\n pass\n', + ]: + with self.subTest(source=source): + result = self.assertFormatting(source) + self.assertIn('# leading', result) + self.assertIn('# trailing', result) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep696_test.py b/yapftests/pep696_test.py new file mode 100644 index 000000000..b72ed2ef1 --- /dev/null +++ b/yapftests/pep696_test.py @@ -0,0 +1,140 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regression coverage for PEP 696, including exact output and stability.""" + +import ast +import sys +import unittest + +from yapf.yapflib import errors +from yapf.yapflib.yapf_api import FormatCode + + +class Pep696Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 13): + self.assertEqual(ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + return result + + def testTypeVariableDefaults(self): + cases = [ + ('class Box[T=int]:\n pass\n', 'class Box[T = int]:\n pass\n'), + ('type Vector[T=int]=list[T]\n', 'type Vector[T = int] = list[T]\n'), + ('def identity[T=int](value:T)->T:\n return value\n', + 'def identity[T = int](value: T) -> T:\n return value\n'), + ('async def identity[T=int](value:T)->T:\n return value\n', + 'async def identity[T = int](value: T) -> T:\n return value\n'), + ('class Box[T:int=int]:\n pass\n', + 'class Box[T: int = int]:\n pass\n'), + ('type Text[T:(str,bytes)=str]=T\n', + 'type Text[T: (str, bytes) = str] = T\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testVariadicDefaults(self): + cases = [ + ('type Shape[* Ts=*tuple[int,str]]=tuple[*Ts]\n', + 'type Shape[*Ts = *tuple[int, str]] = tuple[*Ts]\n'), + ('type Shape[*Ts=Unpack[tuple[int,str]]]=tuple[*Ts]\n', + 'type Shape[*Ts = Unpack[tuple[int, str]]] = tuple[*Ts]\n'), + ('type Callback[** P=[int,str]]=Callable[P,int]\n', + 'type Callback[**P = [int, str]] = Callable[P, int]\n'), + ('type Callback[**P=...]=Callable[P,int]\n', + 'type Callback[**P = ...] = Callable[P, int]\n'), + ('class Box[T,*Ts=*tuple[int],**P=[str]]:\n pass\n', + 'class Box[T, *Ts = *tuple[int], **P = [str]]:\n pass\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testNestedDefaultExpressionsKeepTheirOwnSpacing(self): + cases = [ + ('type Box[T=Factory(option=True)]=list[T]\n', + 'type Box[T = Factory(option=True)] = list[T]\n'), + ('type Box[T=int|None]=list[T]\n', + 'type Box[T = int | None] = list[T]\n'), + ('type Box[T=int if enabled else str]=list[T]\n', + 'type Box[T = int if enabled else str] = list[T]\n'), + ('type Box[T=tuple[int,tuple[str,...]]]=list[T]\n', + 'type Box[T = tuple[int, tuple[str, ...]]] = list[T]\n'), + ('def f[T=int](value=1,*,other:T=2):\n return value\n', + 'def f[T = int](value=1, *, other: T = 2):\n return value\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testTypeDefaultSpacingIsNotKeywordArgumentSpacing(self): + for setting in (False, True): + result = self.assertFormatting( + 'type Box[T=int]=list[T]\n', + style={'SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN': setting}) + self.assertEqual('type Box[T = int] = list[T]\n', result) + + def testWrappedDefaultsCommentsAndTrailingCommas(self): + sources = [ + 'type Alias[\n T: int=int, # default\n ** P=[int,str], # args\n' + ']=Callable[P,T]\n', + 'class Box[LongParameterName=Mapping[str,list[int]],' + 'OtherLongParameterName=tuple[int,str]](Base):\n pass\n', + 'def f[T=Mapping[str,list[int]],** P=[str,int],](' + 'value:T,other:int=0)->T:\n return value\n', + '@decorate\nasync def f[T=int](value:T):\n return value\n', + 'type Shape[*Ts=*tuple[int,str],]=tuple[*Ts]\n', + ] + for source in sources: + for name in ('pep8', 'google', 'yapf', 'facebook'): + for limit in (32, 50, 88): + with self.subTest(source=source, style=name, limit=limit): + result = self.assertFormatting( + source, + style={ + 'BASED_ON_STYLE': name, + 'COLUMN_LIMIT': limit, + 'DEDENT_CLOSING_BRACKETS': True + }) + for comment in ('# default', '# args'): + if comment in source: + self.assertIn(comment, result) + + def testOnlyTupleParameterDefaultsCanBeStarred(self): + for source in ('type A[T=*tuple[int]]=T\n', 'type A[**P=*tuple[int]]=T\n'): + with self.subTest(source=source): + with self.assertRaises(errors.YapfError): + FormatCode(source) + + @unittest.skipUnless(sys.version_info >= (3, 13), 'requires Python 3.13') + def testNativeDefaultMetadata(self): + source = ('class Box[T=int]:\n pass\n' + 'type Callback[**P=[int,str]]=tuple[P]\n') + result = self.assertFormatting(source) + namespace = {} + exec(result, namespace) + self.assertIs(int, namespace['Box'].__type_params__[0].__default__) + self.assertEqual([int, str], + namespace['Callback'].__type_params__[0].__default__) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep701_test.py b/yapftests/pep701_test.py new file mode 100644 index 000000000..ca7a4385a --- /dev/null +++ b/yapftests/pep701_test.py @@ -0,0 +1,133 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PEP 701 regression tests; new syntax stays in string fixtures. + +Native AST comparisons run on Python 3.12 and newer. +""" + +import ast +import io +import sys +import unittest + +from yapf_third_party._ylib2to3.pgen2 import token +from yapf_third_party._ylib2to3.pgen2 import tokenize + +from yapf.yapflib.yapf_api import FormatCode + + +class Pep701Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 12): + self.assertEqual( + ast.dump(ast.parse(source, feature_version=(3, 12))), + ast.dump(ast.parse(result, feature_version=(3, 12)))) + return result + + def testFStringSourceIsPreserved(self): + literals = [ + 'f"{data["key"]}"', + "f'{data['key']}'", + 'f"{f"{f"{value}"}"}"', + 'f"{"\\n".join(items)}"', + 'f"{value # quotes " and } are part of this comment\n}"', + 'f"{value + \\\nother}"', + 'f"{value\n}"', + 'f"{value = !r}"', + 'f"{value:{width # ignored }\n}.{precision}f}"', + 'f"""{data["""key"""]}"""', + 'rf"{{literal}} {data["key"]}"', + 'f"\\N{LEFT CURLY BRACKET}{value}"', + ] + for literal in literals: + with self.subTest(literal=literal): + source = 'result=' + literal + '\n' + self.assertFormatting(source, 'result = ' + literal + '\n') + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + self.assertEqual([literal], + [t[1] for t in tokens if t[0] == token.STRING]) + + def testMultilineFStringFollowedByCode(self): + source = 'result=f"{value # field\n}";other=1\n' + result = self.assertFormatting(source) + self.assertIn('f"{value # field\n}"', result) + self.assertIn('other = 1', result) + source = ('result = [f"{value # field\n}",f"{data["key"]}"]\n' + 'following=2\n') + result = self.assertFormatting(source) + self.assertIn('following = 2', result) + + def testFStringTokenPositions(self): + source = 'result = f"{value # comment\n}"; after = 1\n' + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + literal = next(t for t in tokens if t[0] == token.STRING) + self.assertEqual((1, 9), literal[2]) + self.assertEqual((2, 2), literal[3]) + semicolon = next(t for t in tokens if t[1] == ';') + self.assertEqual((2, 2), semicolon[2]) + + def testMultilineStringRangeFormatting(self): + literals = [ + 'f"{value # comment\n}"', + 'f"""value={value}\n"""', + '"""line\ncontinued"""', + "'line\\\ncontinued'", + ] + for literal in literals: + source = 'before=1\nresult=' + literal + '\nafter=2\n' + expected = 'before=1\nresult = ' + literal + '\nafter=2\n' + for selected_line in (2, 3): + with self.subTest(literal=literal, selected_line=selected_line): + result, changed = FormatCode( + source, + style_config='pep8', + lines=[(selected_line, selected_line)]) + self.assertEqual(expected, result) + self.assertTrue(changed) + again, changed = FormatCode( + result, + style_config='pep8', + lines=[(selected_line, selected_line)]) + self.assertEqual(result, again) + self.assertFalse(changed) + + def testCrLfTokenPositions(self): + source = 'result = f"{value # comment\r\n}"; after = 1\r\n' + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + literal = next(t for t in tokens if t[0] == token.STRING) + self.assertEqual('f"{value # comment\r\n}"', literal[1]) + self.assertEqual((1, 9), literal[2]) + self.assertEqual((2, 2), literal[3]) + self.assertEqual(source, literal[4]) + semicolon = next(t for t in tokens if t[1] == ';') + self.assertEqual((2, 2), semicolon[2]) + + def testMalformedFStringRaisesTokenError(self): + for source in [ + 'f"{value', 'f"{(value]}"', 'f"}"', 'f"{value # missing newline' + ]: + with self.subTest(source=source): + with self.assertRaises(tokenize.TokenError): + list(tokenize.generate_tokens(io.StringIO(source).readline)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep750_test.py b/yapftests/pep750_test.py new file mode 100644 index 000000000..255f20a83 --- /dev/null +++ b/yapftests/pep750_test.py @@ -0,0 +1,144 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regression coverage for PEP 750, including exact output and stability.""" + +import ast +import sys +import unittest +import io +import itertools + +from yapf_third_party._ylib2to3.pgen2 import token +from yapf_third_party._ylib2to3.pgen2 import tokenize + +from yapf.yapflib import errors +from yapf.yapflib.yapf_api import FormatCode + + +class Pep750Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 14): + self.assertEqual(ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + return result + + def testPrefixQuoteAndBodyMatrix(self): + prefixes = ('t', 'T', 'tr', 'tR', 'Tr', 'TR', 'rt', 'rT', 'Rt', 'RT') + quotes = ('"', "'", '"""', "'" * 3) + bodies = ('', 'literal', '{{escaped}}', '{name}', '{name=}', + '{name = !r:>{width}}', '{ {1:2} }') + for prefix, quote, body in itertools.product(prefixes, quotes, bodies): + literal = prefix + quote + body + quote + with self.subTest(literal=literal): + self.assertFormatting('value=' + literal + '\n', + 'value = ' + literal + '\n') + tokens = list(tokenize.generate_tokens(io.StringIO(literal).readline)) + self.assertEqual([literal], + [t[1] for t in tokens if t[0] == token.STRING]) + + def testLexicalContextsAndMixedNesting(self): + literals = [ + 't"{data["key"]}"', + "t'{data['key']}'", + 't"{t"{value}"}"', + 't"{f"{t"{data["key"]}"}"}"', + 'f"{t"{f"{data["key"]}"}"}"', + 't"{"\\n".join(items)}"', + 't"{value # quote " and } in a comment\n}"', + 't"{value + \\\nother}"', + 't"{value:{width # field\n}.{precision}f}"', + 't"\\N{LEFT CURLY BRACKET}{value}"', + 'rt"\\N{value}"', + 't"""{data["""key"""]}"""', + ] + for literal in literals: + for name in ('pep8', 'google', 'yapf', 'facebook'): + with self.subTest(literal=literal, style=name): + result = self.assertFormatting('result=' + literal + '\n', style=name) + self.assertIn(literal, result) + + def testTemplateConcatenationAndExpressionStatements(self): + sources = [ + 'value=t"hello" t"{name}"\n', + 'value=(t"first"\n # comment\n rt"second {name}")\n', + 't"{value}"\nother=1\n', + 'process(t"{a}",t"{b}")\n', + ] + for source in sources: + with self.subTest(source=source): + self.assertFormatting(source) + + def testMultilinePositionsAndFollowingTokens(self): + for newline in ('\n', '\r\n'): + literal = 't"{value # field' + newline + '}"' + source = 'result = ' + literal + '; after = 1' + newline + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + string = next(t for t in tokens if t[0] == token.STRING) + self.assertEqual(literal, string[1]) + self.assertEqual((1, 9), string[2]) + self.assertEqual((2, 2), string[3]) + self.assertEqual(source, string[4]) + semicolon = next(t for t in tokens if t[1] == ';') + self.assertEqual((2, 2), semicolon[2]) + self.assertFormatting('result=t"{value # field\n}";after=1\n') + + def testInteriorLineSelection(self): + literal = 't"{value # field\n}"' + source = 'before=1\nresult=' + literal + '\nafter=2\n' + expected = 'before=1\nresult = ' + literal + '\nafter=2\n' + result, changed = FormatCode(source, lines=[(3, 3)]) + self.assertTrue(changed) + self.assertEqual(expected, result) + self.assertEqual((expected, False), FormatCode(expected, lines=[(3, 3)])) + + def testIdentifiersAreNotStringPrefixes(self): + source = 't=1\nT=2\nrt=3\ntr=4\nresult=t+T+rt+tr\n' + self.assertFormatting( + source, 't = 1\nT = 2\nrt = 3\ntr = 4\n' + 'result = t + T + rt + tr\n') + self.assertFormatting('value=t"{target + starter + rt}"\n') + + def testMalformedLexicalContexts(self): + for literal in ('t"{value', 't"{(value]}"', 't"}"', 't"{x # missing end', + 't"line\nbreak"'): + with self.subTest(literal=literal): + with self.assertRaises(tokenize.TokenError): + list(tokenize.generate_tokens(io.StringIO(literal).readline)) + for prefix in ('ft', 'tf', 'bt', 'tb', 'ut', 'tu', 'rrt', 'trr'): + with self.subTest(prefix=prefix): + with self.assertRaises(errors.YapfError): + FormatCode('value=' + prefix + '"text"\n') + + @unittest.skipUnless(sys.version_info >= (3, 14), 'requires Python 3.14') + def testNativeInterpolationMetadataIsPreserved(self): + source = 'total=6\nmessage=t"{ total + 1 = !r:>{2+1}}"\n' + formatted = self.assertFormatting(source) + before, after = {}, {} + exec(source, before) + exec(formatted, after) + left, right = before['message'], after['message'] + self.assertEqual(left.strings, right.strings) + for a, b in zip(left.interpolations, right.interpolations): + for attribute in ('value', 'expression', 'conversion', 'format_spec'): + self.assertEqual(getattr(a, attribute), getattr(b, attribute)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep758_test.py b/yapftests/pep758_test.py new file mode 100644 index 000000000..f93373405 --- /dev/null +++ b/yapftests/pep758_test.py @@ -0,0 +1,116 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regression coverage for PEP 758, including exact output and stability.""" + +import ast +import sys +import unittest + +from yapf.yapflib import errors +from yapf.yapflib.yapf_api import FormatCode + + +class Pep758Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 14): + self.assertEqual(ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + return result + + def testExceptionListsAndTrailingCommas(self): + for keyword in ('except', 'except*'): + for exceptions, expected in (('A,B', 'A, B'), ('A,B,C', 'A, B, C'), + ('A,', 'A,'), ('A,B,C,', 'A, B, C,'), + ('errors.A,get_errors(),B', + 'errors.A, get_errors(), B')): + source = 'try:\n work()\n%s %s:\n pass\n' % (keyword, exceptions) + target = 'try:\n work()\n%s %s:\n pass\n' % (keyword, expected) + with self.subTest(source=source): + self.assertFormatting(source, target) + + def testExistingParenthesizedAndBareHandlers(self): + for handler in ('except:', 'except Exception:', + 'except Exception as error:', 'except (A, B) as error:', + 'except* (A, B) as error:'): + source = 'try:\n work()\n' + handler + '\n pass\n' + with self.subTest(handler=handler): + self.assertFormatting(source, + 'try:\n work()\n' + handler + '\n pass\n') + + def testCommentsAndContinuations(self): + sources = [ + 'try:\n work()\nexcept A, \\\n B, C: # retained\n pass\n', + 'try:\n work()\nexcept* A, B, C: # retained\n pass\n', + 'try:\n work()\nexcept (\n A, # retained\n B,\n) as errors:\n pass\n', + ] + for source in sources: + with self.subTest(source=source): + result = self.assertFormatting(source) + self.assertIn('# retained', result) + + def testLongListsStaySyntacticallyValid(self): + for keyword in ('except', 'except*'): + source = ('try:\n work()\n' + keyword + ' FirstLongExceptionName,' + 'SecondLongExceptionName,ThirdLongExceptionName:\n pass\n') + for name in ('pep8', 'google', 'yapf', 'facebook'): + with self.subTest(keyword=keyword, style=name): + result = self.assertFormatting( + source, style={ + 'BASED_ON_STYLE': name, + 'COLUMN_LIMIT': 32 + }) + # Without a bracket pair or explicit continuation, a comma alone + # cannot introduce a physical newline in an exception clause. + self.assertIn( + 'FirstLongExceptionName, SecondLongExceptionName, ' + 'ThirdLongExceptionName:', result) + + def testMultipleHandlersElseAndFinally(self): + source = ( + 'try:\n work()\nexcept A,B,C:\n handle()\nexcept D,E:\n handle()\n' + 'else:\n success()\nfinally:\n cleanup()\n') + self.assertFormatting(source) + + def testUnparenthesizedListsCannotBindAnAsTarget(self): + for keyword in ('except', 'except*'): + for exceptions in ('A,B', 'A,B,C', 'A,'): + with self.subTest(keyword=keyword, exceptions=exceptions): + with self.assertRaises(errors.YapfError): + FormatCode('try:\n pass\n%s %s as error:\n pass\n' % + (keyword, exceptions)) + + def testOtherCommaColonSpacingIsUnchanged(self): + self.assertFormatting('value=items[1,:]\n', 'value = items[1, :]\n') + + @unittest.skipUnless(sys.version_info >= (3, 14), 'requires Python 3.14') + def testNativeHandlerSemantics(self): + for keyword, raised in (('except', 'ValueError()'), + ('except*', + 'ExceptionGroup("errors", [ValueError()])')): + source = ('caught=False\ntry:\n raise %s\n' + '%s ValueError,TypeError,RuntimeError:\n caught=True\n' % + (raised, keyword)) + namespace = {} + exec(self.assertFormatting(source), namespace) + self.assertTrue(namespace['caught']) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep798_test.py b/yapftests/pep798_test.py new file mode 100644 index 000000000..93b560fd0 --- /dev/null +++ b/yapftests/pep798_test.py @@ -0,0 +1,134 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regression coverage for PEP 798, including exact output and stability.""" + +import ast +import sys +import unittest + +from yapf.yapflib.yapf_api import FormatCode + + +class Pep798Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 15): + self.assertEqual(ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + return result + + def testAllComprehensionKinds(self): + cases = [ + ('flat=[* items for items in groups]\n', + 'flat = [*items for items in groups]\n'), + ('flat={* items for items in groups}\n', + 'flat = {*items for items in groups}\n'), + ('merged={** mapping for mapping in mappings}\n', + 'merged = {**mapping for mapping in mappings}\n'), + ('flat=(* items for items in groups)\n', + 'flat = (*items for items in groups)\n'), + ('flat=tuple(* items for items in groups)\n', + 'flat = tuple(*items for items in groups)\n'), + ('flat=tuple((* items for items in groups))\n', + 'flat = tuple((*items for items in groups))\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testFullExpressionsInGeneratorAndMappingHeads(self): + sources = [ + 'flat=(*left if enabled else right for left in groups)\n', + 'flat=tuple(*left if enabled else right for left in groups)\n', + 'merged={**left if enabled else right for left in mappings}\n', + 'flat=tuple(*(chosen:=items) for items in groups)\n', + 'flat=tuple(*factory(items) for items in groups if items)\n', + 'flat=tuple(*obj.items[1:] for obj in groups)\n', + ] + for source in sources: + with self.subTest(source=source): + self.assertFormatting(source) + + def testAsyncComprehensions(self): + for expression in ('[*items async for items in groups]', + '{*items async for items in groups}', + '{**items async for items in groups}', + '(*items async for items in groups)', + 'tuple(*items async for items in groups)', + 'tuple(*(await fetch(x)) async for x in groups)'): + source = 'async def flatten(groups):\n return ' + expression + '\n' + with self.subTest(expression=expression): + self.assertFormatting(source) + + def testNestedClausesAndComments(self): + sources = [ + 'flat=tuple(*items for group in groups for items in group if items)\n', + 'flat=tuple(\n *items # retained\n for items in groups if items\n)\n', + 'merged={\n **items # retained\n for items in mappings\n}\n', + 'flat=[*items for items in groups if items if enabled]\n', + ] + for source in sources: + for name in ('pep8', 'google', 'yapf', 'facebook'): + for limit in (30, 50, 88): + with self.subTest(source=source, style=name, limit=limit): + result = self.assertFormatting( + source, + style={ + 'BASED_ON_STYLE': name, + 'COLUMN_LIMIT': limit, + 'DEDENT_CLOSING_BRACKETS': True + }) + if '# retained' in source: + self.assertIn('# retained', result) + + def testExistingUnpackingAndLoopTargets(self): + sources = [ + 'call(*args,**kwargs)\n', + 'value=(*left,*right)\n', + 'value={**left,**right}\n', + 'value=[*left,*right]\n', + 'for *head,tail in items:\n pass\n', + 'head,*tail=items\n', + 'value=call(x for x in values)\n', + 'value=items[*indices]\n', + ] + for source in sources: + with self.subTest(source=source): + result = self.assertFormatting(source) + if sys.version_info >= (3, 11): + self.assertEqual( + ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + + @unittest.skipUnless(sys.version_info >= (3, 15), 'requires Python 3.15') + def testNativeUnpackingSemantics(self): + source = ('groups=[[1,2],[3]]\n' + 'flat=[*items for items in groups]\n' + 'unique={*items for items in groups}\n' + 'generated=tuple(*items for items in groups)\n' + 'merged={**mapping for mapping in [{"x":1},{"x":2,"y":3}]}\n') + namespace = {} + exec(self.assertFormatting(source), namespace) + self.assertEqual([1, 2, 3], namespace['flat']) + self.assertEqual({1, 2, 3}, namespace['unique']) + self.assertEqual((1, 2, 3), namespace['generated']) + self.assertEqual({'x': 2, 'y': 3}, namespace['merged']) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep810_test.py b/yapftests/pep810_test.py new file mode 100644 index 000000000..70180b650 --- /dev/null +++ b/yapftests/pep810_test.py @@ -0,0 +1,143 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regression coverage for PEP 810, including exact output and stability.""" + +import ast +import sys +import unittest +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token + +from yapf.yapflib import format_token + +from yapf.yapflib import errors +from yapf.yapflib.yapf_api import FormatCode + + +class Pep810Test(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 15): + self.assertEqual(ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + return result + + def testLazyImportForms(self): + cases = [ + ('lazy import json\n', 'lazy import json\n'), + ('lazy import a.b as c,d.e as f\n', 'lazy import a.b as c, d.e as f\n'), + ('lazy from package import one as first,two\n', + 'lazy from package import one as first, two\n'), + ('lazy from ..package import one,two\n', + 'lazy from ..package import one, two\n'), + ('lazy from . import one,two\n', 'lazy from . import one, two\n'), + ('lazy import lazy as lazy\n', 'lazy import lazy as lazy\n'), + ('lazy from lazy import lazy as lazy\n', + 'lazy from lazy import lazy as lazy\n'), + ] + for source, expected in cases: + with self.subTest(source=source): + self.assertFormatting(source, expected) + + def testSoftKeywordIsStillAnIdentifier(self): + sources = [ + 'lazy=1\n', + 'lazy(lazy=1)\n', + 'obj.lazy()\n', + 'obj.lazy[0]=lazy\n', + 'def lazy(lazy):\n return lazy\n', + 'class lazy:\n pass\n', + 'from lazy import lazy as lazy\n', + 'match value:\n case {"lazy":lazy}:\n print(lazy)\n', + 'lazy: Callable=other\n', + ] + for source in sources: + with self.subTest(source=source): + result = self.assertFormatting(source) + if sys.version_info >= (3, 10): + self.assertEqual( + ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + leaf = pytree.Leaf(token.NAME, 'lazy') + self.assertFalse(format_token.FormatToken(leaf, 'NAME').is_keyword) + + def testParenthesizedImportsAndComments(self): + sources = [ + 'lazy from package import (first,second,third,)\n', + 'lazy from package import (\n first, # retained\n second,\n)\n', + 'lazy import package # retained\n', + 'lazy from package import first, \\\n second\n', + ] + for source in sources: + for name in ('pep8', 'google', 'yapf', 'facebook'): + with self.subTest(source=source, style=name): + result = self.assertFormatting( + source, + style={ + 'BASED_ON_STYLE': name, + 'COLUMN_LIMIT': 30, + 'DEDENT_CLOSING_BRACKETS': True + }) + if '# retained' in source: + self.assertIn('# retained', result) + + def testMixedEagerLazyAndGuardedImports(self): + sources = [ + 'import sys\nlazy import json\nfrom os import path\n' + 'lazy from collections import Counter\n', + 'if enabled:\n lazy import json\nelse:\n import json\n', + 'if TYPE_CHECKING:\n lazy from package import Something\n', + 'lazy import json;lazy import re\n', + ] + for source in sources: + with self.subTest(source=source): + self.assertFormatting(source) + + def testTopLevelImportVariableSpacing(self): + custom = {'BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES': 2} + for keyword in ('import json', 'from json import dumps', 'lazy import json', + 'lazy from json import dumps'): + self.assertFormatting( + keyword + '\nvalue=1\n', keyword + '\n\n\nvalue = 1\n', style=custom) + # An ordinary name must not become an import just because it is spelled lazy. + self.assertFormatting( + 'lazy=1\nvalue=2\n', 'lazy = 1\nvalue = 2\n', style=custom) + + def testDocstringsAndDefinitions(self): + for declaration in ('def f():\n pass\n', 'class C:\n pass\n'): + source = '"""Documentation."""\nlazy import json\n' + declaration + result = self.assertFormatting(source) + self.assertIn('lazy import json\n\n\n', result) + + def testIncompleteLazyImportsAreRejected(self): + for source in ('lazy import\n', 'lazy from json\n', + 'lazy from json import\n', 'lazy lazy import json\n'): + with self.subTest(source=source): + with self.assertRaises(errors.YapfError): + FormatCode(source) + + @unittest.skipUnless(sys.version_info >= (3, 15), 'requires Python 3.15') + def testNativeLazyImportFlagIsPreserved(self): + source = 'lazy import json\nlazy from collections import Counter\n' + output = self.assertFormatting(source) + for node in ast.parse(output).body: + self.assertTrue(node.is_lazy) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pep_integration_test.py b/yapftests/pep_integration_test.py new file mode 100644 index 000000000..da3754fd6 --- /dev/null +++ b/yapftests/pep_integration_test.py @@ -0,0 +1,103 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regression coverage for the combined PEP features, including exact output and stability.""" + +import ast +import sys +import unittest + +from yapf.yapflib.yapf_api import FormatCode + + +class PepIntegrationTest(unittest.TestCase): + + def assertFormatting(self, source, expected=None, style='pep8'): + result, _ = FormatCode(source, style_config=style) + if expected is not None: + self.assertEqual(expected, result) + again, changed = FormatCode(result, style_config=style) + self.assertEqual(result, again) + self.assertFalse(changed) + if sys.version_info >= (3, 15): + self.assertEqual(ast.dump(ast.parse(source)), ast.dump(ast.parse(result))) + return result + + def testFeaturesComposeInTheSameModule(self): + sources = [ + 'lazy from collections.abc import Iterable\n' + 'def flatten[T=int](groups:Iterable[list[T]])->tuple[T,...]:\n' + ' return tuple(*items for items in groups)\n', + 'type Message[T=int]=tuple[T,str]\nlazy import json\n' + 'result=t"{data["name"]}"\n', + 'try:\n result=tuple(*items for items in groups)\n' + 'except* ValueError,TypeError,RuntimeError:\n' + ' error=t"{info["reason"]}"\n', + 'lazy from types import SimpleNamespace\n' + 'class Collection[T=int](Base[T]):\n' + ' type Member[U=str]=tuple[T,U]\n' + ' def render(self):\n return t"{self.values["first"]}"\n', + 'type Alias[T=type(t"payload")]=list[T]\n', + 'result=t"{[*items for items in groups]}"\n', + ] + for source in sources: + for name in ('pep8', 'google', 'yapf', 'facebook'): + for limit in (32, 79): + with self.subTest(source=source, style=name, limit=limit): + self.assertFormatting( + source, + style={ + 'BASED_ON_STYLE': name, + 'COLUMN_LIMIT': limit, + 'DEDENT_CLOSING_BRACKETS': True + }) + + def testSoftKeywordsDoNotInterfere(self): + sources = [ + 'def lazy[type=int](lazy:type)->type:\n return lazy\n', + 'type lazy[type=int]=list[type]\n', + 'lazy from lazy import type as lazy\n', + 'type type[lazy=int]=list[lazy]\n', + 'lazy(lambda type:type,lazy=type)\n', + ] + for source in sources: + with self.subTest(source=source): + self.assertFormatting(source) + + def testRangeFormattingAcrossNewSyntax(self): + source = ('type Alias[T=int]=list[T]\nresult=t"{value # field\n}"\n' + 'lazy import json\n') + expected = ('type Alias[T=int]=list[T]\nresult = t"{value # field\n}"\n' + 'lazy import json\n') + self.assertEqual((expected, True), FormatCode(source, lines=[(3, 3)])) + self.assertEqual((expected, False), FormatCode(expected, lines=[(3, 3)])) + + def testDisabledRegionKeepsNewSyntaxVerbatim(self): + prefix = ('# yapf: disable\ntype Alias[T=int]=list[T]\n' + 'result=t"{data["key"]}"\n# yapf: enable\n') + source = prefix + 'lazy import json\n' + expected = prefix + 'lazy import json\n' + self.assertFormatting(source, expected) + + def testDiffOutputAndFileName(self): + source = 'type Alias[T=int]=list[T]\nvalue=t"{data["key"]}"\n' + diff, changed = FormatCode( + source, filename='new_syntax.py', print_diff=True) + self.assertTrue(changed) + self.assertIn('new_syntax.py', diff) + self.assertIn('+type Alias[T = int] = list[T]', diff) + self.assertIn('+value = t"{data["key"]}"', diff) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/token_proxy_test.py b/yapftests/token_proxy_test.py new file mode 100644 index 000000000..6d1f0ef4f --- /dev/null +++ b/yapftests/token_proxy_test.py @@ -0,0 +1,144 @@ +# Copyright 2026 The YAPF Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Lookahead must be replayable, nestable, and bounded by unread tokens.""" + +import ast +import sys +import unittest + +from yapf.pytree import pytree_utils +from yapf.yapflib.yapf_api import FormatCode + +TokenProxy = pytree_utils.driver.TokenProxy + + +class TokenProxyTest(unittest.TestCase): + + def testLookaheadDoesNotConsumeOrReadTokensTwice(self): + reads = [] + + def tokens(): + for value in range(6): + reads.append(value) + yield value + + proxy = TokenProxy(tokens()) + self.assertEqual(0, next(proxy)) + with proxy.release(): + self.assertEqual(3, proxy.eat(2)) + self.assertEqual(1, proxy.eat(0)) + self.assertEqual(3, proxy.eat(2)) + self.assertEqual([0, 1, 2, 3], reads) + self.assertEqual([1, 2, 3, 4, 5], list(proxy)) + self.assertEqual(list(range(6)), reads) + + def testNestedLookaheadRestoresParentCursor(self): + proxy = TokenProxy(iter(range(6))) + self.assertEqual(0, next(proxy)) + with proxy.release(): + self.assertEqual(1, proxy.eat(0)) + with proxy.release(): + self.assertEqual(2, proxy.eat(0)) + self.assertEqual(3, proxy.eat(1)) + self.assertEqual(2, proxy.eat(1)) + with proxy.release(): + self.assertEqual(3, proxy.eat(0)) + self.assertEqual(3, proxy.eat(2)) + self.assertEqual([1, 2, 3, 4, 5], list(proxy)) + + def testCompetingLookaheadsSeeTheSameStream(self): + proxy = TokenProxy(iter(range(8))) + next(proxy) + with proxy.release(): + self.assertEqual(1, proxy.eat(0)) + for _ in range(3): + with proxy.release(): + self.assertEqual(2, proxy.eat(0)) + self.assertEqual(5, proxy.eat(3)) + self.assertEqual(2, proxy.eat(1)) + self.assertEqual(list(range(1, 8)), list(proxy)) + + def testUnwindingAfterErrorPreservesBufferedTokens(self): + proxy = TokenProxy(iter(range(5))) + with self.assertRaises(ValueError): + with proxy.release(): + proxy.eat(0) + with proxy.release(): + proxy.eat(2) + raise ValueError('rejected parser route') + self.assertEqual(list(range(5)), list(proxy)) + self.assertFalse(proxy._lookahead) + + def testLookaheadPastEofDoesNotLoseAvailableTokens(self): + proxy = TokenProxy(iter(range(3))) + with proxy.release(): + self.assertFalse(proxy.can_advance(8)) + self.assertTrue(proxy.can_advance(2)) + self.assertEqual([0, 1, 2], list(proxy)) + + def testConsumedTokensAndScopesAreDiscarded(self): + # A structural complexity check, not a timing assertion. The old proxy + # retained one scope per keyword and rescanned all scopes for every token. + proxy = TokenProxy(iter(range(4000))) + for value in range(4000): + with proxy.release(): + self.assertEqual(value, proxy.eat(0)) + self.assertEqual(value, next(proxy)) + self.assertFalse(proxy._buffer) + self.assertFalse(proxy._lookahead) + + def testNegativeLookaheadIsRejected(self): + proxy = TokenProxy(iter(range(2))) + with proxy.release(): + with self.assertRaises(IndexError): + proxy.eat(-1) + + +class NestedSoftKeywordTest(unittest.TestCase): + + def testSoftKeywordsInsideAmbiguousMatchExpressions(self): + for name in ('match', 'case', 'type', 'lazy'): + for subject in (name, name + '(value)', name + ', other', + 'lambda ' + name + ': ' + name): + for brackets in ('(%s)', '[%s]'): + expression = brackets % subject + for source in ('match' + expression + '\n', + 'match ' + expression + ':\n case _: pass\n'): + with self.subTest(source=source): + output, _ = FormatCode(source, style_config='pep8') + self.assertEqual((output, False), + FormatCode(output, style_config='pep8')) + if sys.version_info >= (3, 10): + self.assertEqual( + ast.dump(ast.parse(source)), ast.dump(ast.parse(output))) + + def testNestedAmbiguityAndComments(self): + sources = [ + 'match(match(type), lazy, case)\n', + 'match[match[case]]\n', + 'match (type # comment\n, lazy):\n case _: pass\n', + 'match (match(case),type(lazy)):\n case _: pass\n', + ] + for source in sources: + with self.subTest(source=source): + output, _ = FormatCode(source, style_config='pep8') + self.assertEqual((output, False), + FormatCode(output, style_config='pep8')) + if sys.version_info >= (3, 10): + self.assertEqual( + ast.dump(ast.parse(source)), ast.dump(ast.parse(output))) + + +if __name__ == '__main__': + unittest.main()