Fix .gitignore exclusion list build to include folders starting with a dot (#6978)

* Fix .gitignore exclusion list build to include folders starting with a dot

* Improve performance of listing .gitignored files and advise on additional exclusions

* yarn audit fix

* cspell

* flake

* uv dep

* [MegaLinter] Apply linters fixes

---------

Co-authored-by: nvuillam <[email protected]>

Author: nvuillam

.cspell.json

@@ -1647,6 +1647,7 @@
         "whitespaces",
         "windir",
         "wipeverb",
+        "wireit",
         "workdir",
         "workerpool",
         "wrappy",

CHANGELOG.md

@@ -11,7 +11,8 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l
 - Core
   - Improve files browsing performances
   - Optimize parallel linter processing and improve grouping logic
-  - Improve performance of listing .gitignored files by sending excluded directories to git ls-files 
+  - Improve performance of listing .gitignored files by sending excluded directories to git ls-files
+  - If there are more than 500 .gitignored files, advise to add more excluded directories using variable ADDITIONAL_EXCLUDED_DIRECTORIES, to improve performances
 
 - New linters
 

README.md

@@ -1125,6 +1125,18 @@ Examples:
 
 Warning: Not applicable with linters using CLI lint mode `project` ([see details](https://github.com/oxsecurity/megalinter/tree/main/docs/config-cli-lint-mode.md)).
 
+You can also use variable **ADDITIONAL_EXCLUDED_DIRECTORIES** to add more excluded directories
+
+Example:
+
+```yaml
+ADDITIONAL_EXCLUDED_DIRECTORIES:
+  - .wireit
+  - dist
+  - build
+  - vendor
+```
+
 <!-- config-filtering-section-end -->
 <!-- config-apply-fixes-section-start -->
 <!-- markdown-headers

mega-linter-runner/yarn.lock

@@ -3365,9 +3365,9 @@ supports-color@^8.1.1:
     has-flag "^4.0.0"
 
 tar@^7.4.3, tar@^7.5.2:
-  version "7.5.2"
-  resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.2.tgz#115c061495ec51ff3c6745ff8f6d0871c5b1dedc"
-  integrity sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.3.tgz#e1a41236e32446f75e63b720222112c4ffe5b3a1"
+  integrity sha512-ENg5JUHUm2rDD7IvKNFGzyElLXNjachNLp6RaGf4+JOgxXHkqA+gq81ZAMCUmtMtqBsoU62lcp6S27g1LCYGGQ==
   dependencies:
     "@isaacs/fs-minipass" "^4.0.0"
     chownr "^3.0.0"

megalinter/MegaLinter.py

@@ -869,11 +869,17 @@ def list_git_ignored_files(self):
         dirpath = os.path.realpath(self.github_workspace)
         repo = git.Repo(dirpath)
         excluded_dirs = utils.get_excluded_directories(self.request_id)
-        normalized_excluded_dirs = set(
-            os.path.normpath(excluded_dir).replace("\\", "/").lstrip("./")
-            for excluded_dir in excluded_dirs
-        )
-        pathspec_excludes = [f":(exclude){excluded_dir}/**" for excluded_dir in normalized_excluded_dirs if excluded_dir]
+        normalized_excluded_dirs = set()
+        for excluded_dir in excluded_dirs:
+            normalized = os.path.normpath(excluded_dir).replace("\\", "/")
+            if normalized.startswith("./"):
+                normalized = normalized[2:]
+            normalized_excluded_dirs.add(normalized)
+        pathspec_excludes = [
+            f":(exclude){excluded_dir}/**"
+            for excluded_dir in normalized_excluded_dirs
+            if excluded_dir
+        ]
         ignored_files = repo.git.execute(
             [
                 "git",
@@ -887,6 +893,15 @@ def list_git_ignored_files(self):
         ).splitlines()
         ignored_files = map(lambda x: x + "**" if x.endswith("/") else x, ignored_files)
         ignored_files = sorted(list(ignored_files))
+        # If there are more than 500 ignored files, advise to add more excluded
+        # directories using variable ADDITIONAL_EXCLUDED_DIRECTORIES, to improve performances
+        if len(ignored_files) > 300:
+            logging.warning(
+                f"⚠️ More than 300 .gitignored files have been detected ({len(ignored_files)}). "
+                "To improve MegaLinter performances, consider adding more excluded directories "
+                "using the ADDITIONAL_EXCLUDED_DIRECTORIES variable. "
+                f"See {ML_DOC_URL}/config-filtering/"
+            )
         return ignored_files
 
     def initialize_output(self):