From c842c9433b2c134ccfe351200864687cf3a41c76 Mon Sep 17 00:00:00 2001 From: Midas <280795179+KaizenShogun@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:17:59 +0200 Subject: [PATCH 1/2] Do not apply the excluded directory rule to a re-included ancestor The directory bucket now only accepts matches on a strict ancestor, and the rule only fires once the ancestor is confirmed excluded by the whole spec, asked outermost first. Fixes part B of #137. --- CHANGES.rst | 1 + pathspec/_backends/hyperscan/gitignore.py | 33 +++++++++++++++--- pathspec/_backends/re2/gitignore.py | 33 +++++++++++++++--- pathspec/_backends/simple/gitignore.py | 42 +++++++++++++++++++++-- tests/test_06_gitignore.py | 32 +++++++++++++++++ 5 files changed, 128 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6ba4c74..f835382 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,6 +27,7 @@ Bug fixes: - `Pull #135`_: Escape trailing spaces in `GitIgnoreSpecPattern.escape()`. - `Issue #137`_ / `Pull #138`_: Patterns ending in `/**` no longer match their bare parent directory, preserving traversal to re-included children. - `Pull #139`_: Match newline characters in paths with `*` and `**`. +- `Issue #137`_: The excluded directory rule no longer applies to an ancestor directory which the spec itself re-includes. .. _`Issue #116`: https://github.com/cpburnz/python-pathspec/issues/116 diff --git a/pathspec/_backends/hyperscan/gitignore.py b/pathspec/_backends/hyperscan/gitignore.py index 8d3c323..be3a973 100644 --- a/pathspec/_backends/hyperscan/gitignore.py +++ b/pathspec/_backends/hyperscan/gitignore.py @@ -131,14 +131,17 @@ def _init_db( # Found directory marker. if regex_str.endswith(_DIR_MARK_OPT): # Regex has optional directory marker. Split regex into directory - # and file variants. + # and file variants. The directory variant must require a child + # segment: without one the query *is* that directory, which is a + # direct match and not an excluded ancestor. base_regex = regex_str[:-len(_DIR_MARK_OPT)] - use_regexes.append((f'{base_regex}/', True)) - use_regexes.append((f'{base_regex}$', False)) + use_regexes.append((f'{base_regex}/(?s:.)', True)) + use_regexes.append((f'{base_regex}/?$', False)) else: # Remove capture group. base_regex = regex_str.replace(_DIR_MARK_CG, '/') - use_regexes.append((base_regex, True)) + use_regexes.append((f'{base_regex}(?s:.)', True)) + use_regexes.append((f'{base_regex}$', False)) if not use_regexes: # No special case for regex. @@ -205,15 +208,35 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: db.scan(file.encode('utf8'), match_event_handler=self.__on_match) dir_include, dir_index, file_include, file_index = self._out - if dir_include: + if dir_include and self._ancestor_excluded(file): out_include, out_index = dir_include, dir_index elif file_include is not None: out_include, out_index = file_include, file_index + elif dir_include: + # An ancestor matched an exclude pattern, but the spec as a whole + # re-includes that ancestor, so the rule does not apply. + out_include, out_index = None, -1 else: out_include, out_index = dir_include, dir_index return (out_include, out_index if out_index != -1 else None) + def _ancestor_excluded(self, file: str) -> bool: + """ + Whether any strict ancestor directory of *file* is excluded. Git stops + descending at the first excluded directory, so the ancestors are asked + outermost first, each as a directory query (trailing slash included). + """ + index = file.find('/') + while index != -1 and index + 1 < len(file): + ancestor_include, _ancestor_index = self.match_file(file[:index + 1]) + if ancestor_include: + return True + index = file.find('/', index + 1) + + return False + + @override def __on_match( self, diff --git a/pathspec/_backends/re2/gitignore.py b/pathspec/_backends/re2/gitignore.py index 987c1e6..7e2b9cd 100644 --- a/pathspec/_backends/re2/gitignore.py +++ b/pathspec/_backends/re2/gitignore.py @@ -97,14 +97,17 @@ def _init_set( # Found directory marker. if regex_str.endswith(_DIR_MARK_OPT): # Regex has optional directory marker. Split regex into directory - # and file variants. + # and file variants. The directory variant must require a child + # segment: without one the query *is* that directory, which is a + # direct match and not an excluded ancestor. base_regex = regex_str[:-len(_DIR_MARK_OPT)] - use_regexes.append((f'{base_regex}/', True)) - use_regexes.append((f'{base_regex}$', False)) + use_regexes.append((f'{base_regex}/(?s:.)', True)) + use_regexes.append((f'{base_regex}/?$', False)) else: # Remove capture group. base_regex = regex_str.replace(_DIR_MARK_CG, '/') - use_regexes.append((base_regex, True)) + use_regexes.append((f'{base_regex}(?s:.)', True)) + use_regexes.append((f'{base_regex}$', False)) if not use_regexes: # No special case for regex. @@ -173,9 +176,29 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: file_index = index assert dir_index != -1 or file_index != -1, (dir_index, file_index) - if dir_include: + if dir_include and self._ancestor_excluded(file): return (dir_include, dir_index) elif file_include is not None: return (file_include, file_index) + elif dir_include: + # An ancestor matched an exclude pattern, but the spec as a whole + # re-includes that ancestor, so the rule does not apply. + return (None, None) else: return (dir_include, dir_index) + + def _ancestor_excluded(self, file: str) -> bool: + """ + Whether any strict ancestor directory of *file* is excluded. Git stops + descending at the first excluded directory, so the ancestors are asked + outermost first, each as a directory query (trailing slash included). + """ + index = file.find('/') + while index != -1 and index + 1 < len(file): + ancestor_include, _ancestor_index = self.match_file(file[:index + 1]) + if ancestor_include: + return True + index = file.find('/', index + 1) + + return False + diff --git a/pathspec/_backends/simple/gitignore.py b/pathspec/_backends/simple/gitignore.py index 94fed90..1f875c5 100644 --- a/pathspec/_backends/simple/gitignore.py +++ b/pathspec/_backends/simple/gitignore.py @@ -78,18 +78,54 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: ): # Pattern matched. if match.match.groupdict().get(_DIR_MARK): - # Pattern matched by a directory pattern. - if dir_include is None or not is_reversed: + # A pattern can match both a strict ancestor of the file and the + # file itself, and the engine only ever hands back the leftmost + # match. Ask for every directory separator it can match. + is_ancestor = is_self = False + assert pattern.regex is not None, pattern + for dir_match in pattern.regex.finditer(file): + if dir_match.groupdict().get(_DIR_MARK) is None: + continue + elif dir_match.end(_DIR_MARK) < len(file): + is_ancestor = True + else: + is_self = True + + if is_ancestor and (dir_include is None or not is_reversed): dir_include = include dir_index = index + + if is_self and (file_include is None or not is_reversed): + file_include = include + file_index = index elif file_include is None or not is_reversed: # Pattern matched by a file pattern. file_include = include file_index = index - if dir_include: + if dir_include and self._ancestor_excluded(file): return (dir_include, dir_index) elif file_include is not None: return (file_include, file_index) + elif dir_include: + # An ancestor matched an exclude pattern, but the spec as a whole + # re-includes that ancestor, so the rule does not apply. + return (None, None) else: return (dir_include, dir_index) + + def _ancestor_excluded(self, file: str) -> bool: + """ + Whether any strict ancestor directory of *file* is excluded. Git stops + descending at the first excluded directory, so the ancestors are asked + outermost first, each as a directory query (trailing slash included). + """ + index = file.find('/') + while index != -1 and index + 1 < len(file): + ancestor_include, _ancestor_index = self.match_file(file[:index + 1]) + if ancestor_include: + return True + index = file.find('/', index + 1) + + return False + diff --git a/tests/test_06_gitignore.py b/tests/test_06_gitignore.py index aea7ec7..8cf5c88 100644 --- a/tests/test_06_gitignore.py +++ b/tests/test_06_gitignore.py @@ -955,3 +955,35 @@ def test_13_issue_139(self): for sub_test in self.parameterize_from_lines([pattern]): with sub_test() as spec: self.assertTrue(spec.match_file(path)) + + def test_14_issue_137_b(self): + """ + Test that the excluded ancestor rule does not fire on an ancestor which + the spec itself re-includes. + """ + for sub_test in self.parameterize_from_lines([ + ".*", + "!**/node_modules/**", + ]): + with sub_test() as spec: + # Confirmed results with git (v2.55.0). Asked two ways which + # agree on every row: "check-ignore -v" (which also prints a + # path whose deciding pattern is a negation, so the pattern + # column is what answers), and the consequence of "git add -A", + # which stages exactly the files that are not ignored. + files = { + ".hidden", # 1:.* + "vendor/keep.txt", # - + "vendor/.cache/y.txt", # 1:.* + "vendor/deps/npm/node_modules/keep.txt", # 2:!**/node_modules/** + "vendor/deps/npm/node_modules/.bin/x.txt", # 2:!**/node_modules/** + "vendor/deps/npm/node_modules/.bin/.hide.txt", # 2:!**/node_modules/** + "node_modules/.bin/z.txt", # 2:!**/node_modules/** + } + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { + ".hidden", + "vendor/.cache/y.txt", + }, debug) From f5c2fcdc72d6e69fec3c47a8ba71cea8681737ae Mon Sep 17 00:00:00 2001 From: KaizenShogun Date: Tue, 29 Sep 2026 13:31:29 +0200 Subject: [PATCH 2/2] Don't re-check an ancestor's ancestors in _ancestor_excluded() Ancestors are asked outermost first, so when one is asked every ancestor above it is already known not to be excluded. Asking again through match_file() made each level re-ask all the levels above it: with ['d0/', '!/d0/', 'd0/', '!d*/', 'd0/d1/', '!d*/**'] a path 20 levels deep took 1,048,576 match calls; it now takes 21. Also drop the unreachable None check in the simple backend's finditer loop, and note why the loop is needed: '**/' compiles to the unanchored '(?P/)', which matches both an ancestor and the directory itself. Co-Authored-By: Claude Opus 5.5 --- pathspec/_backends/hyperscan/gitignore.py | 20 ++++++++++++++-- pathspec/_backends/re2/gitignore.py | 20 ++++++++++++++-- pathspec/_backends/simple/gitignore.py | 29 +++++++++++++++++------ 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/pathspec/_backends/hyperscan/gitignore.py b/pathspec/_backends/hyperscan/gitignore.py index be3a973..133b5cc 100644 --- a/pathspec/_backends/hyperscan/gitignore.py +++ b/pathspec/_backends/hyperscan/gitignore.py @@ -196,6 +196,16 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: or :data:`None`), and the index of the last matched pattern (:class:`int` or :data:`None`). """ + return self._match(file, check_ancestors=True) + + def _match( + self, file: str, check_ancestors: bool, + ) -> tuple[Optional[bool], Optional[int]]: + """ + Implements :meth:`match_file`. *check_ancestors* (:class:`bool`) is + whether to ask if an ancestor directory of *file* is excluded; see + :meth:`_ancestor_excluded` for why it can be skipped. + """ # NOTICE: According to benchmarking, a method callback is 13% faster than # using a closure here. db = self._db @@ -208,7 +218,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: db.scan(file.encode('utf8'), match_event_handler=self.__on_match) dir_include, dir_index, file_include, file_index = self._out - if dir_include and self._ancestor_excluded(file): + if dir_include and check_ancestors and self._ancestor_excluded(file): out_include, out_index = dir_include, dir_index elif file_include is not None: out_include, out_index = file_include, file_index @@ -226,10 +236,16 @@ def _ancestor_excluded(self, file: str) -> bool: Whether any strict ancestor directory of *file* is excluded. Git stops descending at the first excluded directory, so the ancestors are asked outermost first, each as a directory query (trailing slash included). + By the time an ancestor is asked, every ancestor above it is known not to + be excluded, so it is matched without checking its own ancestors again; + otherwise each level re-asks all the levels above it and the work grows + exponentially with depth. """ index = file.find('/') while index != -1 and index + 1 < len(file): - ancestor_include, _ancestor_index = self.match_file(file[:index + 1]) + ancestor_include, _ancestor_index = self._match( + file[:index + 1], check_ancestors=False, + ) if ancestor_include: return True index = file.find('/', index + 1) diff --git a/pathspec/_backends/re2/gitignore.py b/pathspec/_backends/re2/gitignore.py index 7e2b9cd..de8b15f 100644 --- a/pathspec/_backends/re2/gitignore.py +++ b/pathspec/_backends/re2/gitignore.py @@ -145,6 +145,16 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: or :data:`None`), and the index of the last matched pattern (:class:`int` or :data:`None`). """ + return self._match(file, check_ancestors=True) + + def _match( + self, file: str, check_ancestors: bool, + ) -> tuple[Optional[bool], Optional[int]]: + """ + Implements :meth:`match_file`. *check_ancestors* (:class:`bool`) is + whether to ask if an ancestor directory of *file* is excluded; see + :meth:`_ancestor_excluded` for why it can be skipped. + """ # Find best match. match_ids: Optional[list[int]] = self._set.Match(file) # type: ignore[assignment] if not match_ids: @@ -176,7 +186,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: file_index = index assert dir_index != -1 or file_index != -1, (dir_index, file_index) - if dir_include and self._ancestor_excluded(file): + if dir_include and check_ancestors and self._ancestor_excluded(file): return (dir_include, dir_index) elif file_include is not None: return (file_include, file_index) @@ -192,10 +202,16 @@ def _ancestor_excluded(self, file: str) -> bool: Whether any strict ancestor directory of *file* is excluded. Git stops descending at the first excluded directory, so the ancestors are asked outermost first, each as a directory query (trailing slash included). + By the time an ancestor is asked, every ancestor above it is known not to + be excluded, so it is matched without checking its own ancestors again; + otherwise each level re-asks all the levels above it and the work grows + exponentially with depth. """ index = file.find('/') while index != -1 and index + 1 < len(file): - ancestor_include, _ancestor_index = self.match_file(file[:index + 1]) + ancestor_include, _ancestor_index = self._match( + file[:index + 1], check_ancestors=False, + ) if ancestor_include: return True index = file.find('/', index + 1) diff --git a/pathspec/_backends/simple/gitignore.py b/pathspec/_backends/simple/gitignore.py index 1f875c5..0723ab0 100644 --- a/pathspec/_backends/simple/gitignore.py +++ b/pathspec/_backends/simple/gitignore.py @@ -62,6 +62,16 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: or :data:`None`), and the index of the last matched pattern (:class:`int` or :data:`None`). """ + return self._match(file, check_ancestors=True) + + def _match( + self, file: str, check_ancestors: bool, + ) -> tuple[Optional[bool], Optional[int]]: + """ + Implements :meth:`match_file`. *check_ancestors* (:class:`bool`) is + whether to ask if an ancestor directory of *file* is excluded; see + :meth:`_ancestor_excluded` for why it can be skipped. + """ is_reversed = self._is_reversed # Resolve the ancestor directory and the file separately: a file negation @@ -79,14 +89,13 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: # Pattern matched. if match.match.groupdict().get(_DIR_MARK): # A pattern can match both a strict ancestor of the file and the - # file itself, and the engine only ever hands back the leftmost - # match. Ask for every directory separator it can match. + # file itself. For anchored patterns there is only one match, but + # `**/` compiles to the unanchored `(?P/)`: on "a/b/" it + # matches at "a/" and at "b/", and `match` only returns the first. is_ancestor = is_self = False assert pattern.regex is not None, pattern for dir_match in pattern.regex.finditer(file): - if dir_match.groupdict().get(_DIR_MARK) is None: - continue - elif dir_match.end(_DIR_MARK) < len(file): + if dir_match.end(_DIR_MARK) < len(file): is_ancestor = True else: is_self = True @@ -103,7 +112,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]: file_include = include file_index = index - if dir_include and self._ancestor_excluded(file): + if dir_include and check_ancestors and self._ancestor_excluded(file): return (dir_include, dir_index) elif file_include is not None: return (file_include, file_index) @@ -119,10 +128,16 @@ def _ancestor_excluded(self, file: str) -> bool: Whether any strict ancestor directory of *file* is excluded. Git stops descending at the first excluded directory, so the ancestors are asked outermost first, each as a directory query (trailing slash included). + By the time an ancestor is asked, every ancestor above it is known not to + be excluded, so it is matched without checking its own ancestors again; + otherwise each level re-asks all the levels above it and the work grows + exponentially with depth. """ index = file.find('/') while index != -1 and index + 1 < len(file): - ancestor_include, _ancestor_index = self.match_file(file[:index + 1]) + ancestor_include, _ancestor_index = self._match( + file[:index + 1], check_ancestors=False, + ) if ancestor_include: return True index = file.find('/', index + 1)