Properly implemented new filter_glob and exclude_glob parameters in fs.walk.

To do so, fs.wildcard had to undergo a rework to remove the dependency on the `re` module.
Unit tests were added for all new/changed code.

Author: atollk

fs/_ftp_parse.py

@@ -78,7 +78,7 @@ def parse(lines):
 
 def parse_line(line):
     for line_re, decode_callable in get_decoders():
-        match = line_re.match(line)
+        match = line_re._match(line)
         if match is not None:
             return decode_callable(line, match)
     return None

fs/base.py

@@ -539,22 +539,22 @@ def filterdir(
         def match_dir(patterns, info):
             # type: (Optional[Iterable[Text]], Info) -> bool
             """Pattern match info.name."""
-            return info.is_file or self.match(patterns, info.name)
+            return info.is_file or self._match(patterns, info.name)
 
         def match_file(patterns, info):
             # type: (Optional[Iterable[Text]], Info) -> bool
             """Pattern match info.name."""
-            return info.is_dir or self.match(patterns, info.name)
+            return info.is_dir or self._match(patterns, info.name)
 
         def exclude_dir(patterns, info):
             # type: (Optional[Iterable[Text]], Info) -> bool
             """Pattern match info.name."""
-            return info.is_file or not self.match(patterns, info.name)
+            return info.is_file or not self._match(patterns, info.name)
 
         def exclude_file(patterns, info):
             # type: (Optional[Iterable[Text]], Info) -> bool
             """Pattern match info.name."""
-            return info.is_dir or not self.match(patterns, info.name)
+            return info.is_dir or not self._match(patterns, info.name)
 
         if files:
             filters.append(partial(match_file, files))
@@ -1543,13 +1543,16 @@ def check(self):
         if self.isclosed():
             raise errors.FilesystemClosed()
 
-    def match(self, patterns, name):
-        # type: (Optional[Iterable[Text]], Text) -> bool
+    def match(self, patterns, name, accept_prefix=False):
+        # type: (Optional[Iterable[Text]], Text, bool) -> bool
         """Check if a name matches any of a list of wildcards.
 
         Arguments:
             patterns (list): A list of patterns, e.g. ``['*.py']``
             name (str): A file or directory name (not a path)
+            accept_prefix (bool): If ``True``, the name is
+                not required to match the wildcards themselves
+                but only need to be a prefix of a string that does.
 
         Returns:
             bool: `True` if ``name`` matches any of the patterns.
@@ -1561,9 +1564,9 @@ def match(self, patterns, name):
         names).
 
         Example:
-            >>> home_fs.match(['*.py'], '__init__.py')
+            >>> home_fs._match(['*.py'], '__init__.py')
             True
-            >>> home_fs.match(['*.jpg', '*.png'], 'foo.gif')
+            >>> home_fs._match(['*.jpg', '*.png'], 'foo.gif')
             False
 
         Note:
@@ -1578,7 +1581,7 @@ def match(self, patterns, name):
         case_sensitive = not typing.cast(
             bool, self.getmeta().get("case_insensitive", False)
         )
-        matcher = wildcard.get_matcher(patterns, case_sensitive)
+        matcher = wildcard.get_matcher(patterns, case_sensitive, accept_prefix=accept_prefix)
         return matcher(name)
 
     def tree(self, **kwargs):

fs/test.py

@@ -1783,9 +1783,9 @@ def test_movedir(self):
             self.fs.movedir("foo2/foofoo.txt", "foo2/baz/egg")
 
     def test_match(self):
-        self.assertTrue(self.fs.match(["*.py"], "foo.py"))
+        self.assertTrue(self.fs._match(["*.py"], "foo.py"))
         self.assertEqual(
-            self.fs.match(["*.py"], "FOO.PY"),
+            self.fs._match(["*.py"], "FOO.PY"),
             self.fs.getmeta().get("case_insensitive", False),
         )
 

fs/walk.py

@@ -97,12 +97,12 @@ def __init__(
             max_depth (int, optional): Maximum directory depth to walk.
             filter_glob (list, optional): If supplied, this parameter
                 should be a list of path patterns e.g. ``["foo/**/*.py"]``.
-                Resources will only be returned if their global patah
-                matches one of the patterns.
+                Resources will only be returned if their global path or
+                an extension of it matches one of the patterns.
             exclude_glob (list, optional): If supplied, this parameter
                 should be a list of path patterns e.g. ``["foo/**/*.py"]``.
-                Resources will only be returned if their global patah
-                matches one of the patterns.
+                Resources will not be returned if their global path or
+                an extension of it  matches one of the patterns.
 
         """
         if search not in ("breadth", "depth"):
@@ -215,13 +215,18 @@ def _iter_walk(
     def _check_open_dir(self, fs, path, info):
         # type: (FS, Text, Info) -> bool
         """Check if a directory should be considered in the walk."""
+        full_path = ("" if path == "/" else path) + "/" + info.name
         if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name):
             return False
-        if self.exclude_glob is not None and fs.match(self.exclude_glob, path):
+        if self.exclude_glob is not None and fs.match(self.exclude_glob, full_path):
             return False
-        if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name):
+        if self.filter_dirs is not None and not fs.match(
+            self.filter_dirs, info.name, accept_prefix=True
+        ):
             return False
-        if self.filter_glob is not None and not fs.match(self.filter_glob, path):
+        if self.filter_glob is not None and not fs.match(
+            self.filter_glob, full_path, accept_prefix=True
+        ):
             return False
         return self.check_open_dir(fs, path, info)
 
@@ -268,15 +273,23 @@ def check_scan_dir(self, fs, path, info):
         """
         return True
 
-    def _check_file(self, fs, info):
-        # type: (FS, Info) -> bool
+    def _check_file(self, fs, dir_path, info):
+        # type: (FS, Text, Info) -> bool
         """Check if a filename should be included."""
         # Weird check required for backwards compatibility, when _check_file did not exist.
         if Walker._check_file == type(self)._check_file:
             if self.exclude is not None and fs.match(self.exclude, info.name):
                 return False
+            if self.exclude_glob is not None and fs.match(
+                self.exclude_glob, dir_path + "/" + info.name
+            ):
+                return False
             if self.filter is not None and not fs.match(self.filter, info.name):
                 return False
+            if self.filter_glob is not None and not fs.match(
+                self.filter_glob, dir_path + "/" + info.name, accept_prefix=True
+            ):
+                return False
         return self.check_file(fs, info)
 
     def check_file(self, fs, info):
@@ -462,7 +475,7 @@ def _walk_breadth(
                         if _check_scan_dir(fs, dir_path, info, _depth):
                             push(_combine(dir_path, info.name))
                 else:
-                    if _check_file(fs, info):
+                    if _check_file(fs, dir_path, info):
                         yield dir_path, info  # Found a file
             yield dir_path, None  # End of directory
 
@@ -513,7 +526,7 @@ def _walk_depth(
                     else:
                         yield dir_path, info
             else:
-                if _check_file(fs, info):
+                if _check_file(fs, dir_path, info):
                     yield dir_path, info