Skip to content

Commit c93fc98

Browse files
Add ecosystem specific inclusions or exclusions (#1550)
* Add ecosystem specific inclusions or exclusions Also ignore specific files paths containing metadata in ruby gems. Reference: #1438 Reference: #1476 Signed-off-by: Ayan Sinha Mahapatra <[email protected]> Signed-off-by: Philippe Ombredanne <[email protected]> Co-authored-by: Philippe Ombredanne <[email protected]>
1 parent a018711 commit c93fc98

File tree

10 files changed

+305
-45
lines changed

10 files changed

+305
-45
lines changed

scanpipe/pipelines/__init__.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,10 @@ def flag_ignored_resources(self):
7878
ignored_patterns = ignored_patterns.splitlines()
7979
ignored_patterns.extend(flag.DEFAULT_IGNORED_PATTERNS)
8080

81-
flag.flag_ignored_patterns(self.project, patterns=ignored_patterns)
81+
flag.flag_ignored_patterns(
82+
codebaseresources=self.project.codebaseresources.no_status(),
83+
patterns=ignored_patterns,
84+
)
8285

8386
def extract_archive(self, location, target):
8487
"""Extract archive at `location` to `target`. Save errors as messages."""
@@ -179,6 +182,8 @@ def __init__(self, run_instance):
179182
self.selected_groups = run_instance.selected_groups
180183
self.selected_steps = run_instance.selected_steps
181184

185+
self.ecosystem_config = None
186+
182187
@classmethod
183188
def get_initial_steps(cls):
184189
"""Add the ``download_inputs`` step as an initial step if enabled."""

scanpipe/pipelines/deploy_to_develop.py

Lines changed: 22 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from scanpipe import pipes
2525
from scanpipe.pipelines import Pipeline
2626
from scanpipe.pipes import d2d
27+
from scanpipe.pipes import d2d_config
2728
from scanpipe.pipes import flag
2829
from scanpipe.pipes import input
2930
from scanpipe.pipes import matchcode
@@ -64,6 +65,8 @@ def steps(cls):
6465
cls.flag_empty_files,
6566
cls.flag_whitespace_files,
6667
cls.flag_ignored_resources,
68+
cls.load_ecosystem_config,
69+
cls.map_ruby,
6770
cls.map_about_files,
6871
cls.map_checksum,
6972
cls.match_archives_to_purldb,
@@ -94,33 +97,6 @@ def steps(cls):
9497
cls.create_local_files_packages,
9598
)
9699

97-
purldb_package_extensions = [".jar", ".war", ".zip"]
98-
purldb_resource_extensions = [
99-
".map",
100-
".js",
101-
".mjs",
102-
".ts",
103-
".d.ts",
104-
".jsx",
105-
".tsx",
106-
".css",
107-
".scss",
108-
".less",
109-
".sass",
110-
".soy",
111-
".class",
112-
]
113-
doc_extensions = [
114-
".pdf",
115-
".doc",
116-
".docx",
117-
".ppt",
118-
".pptx",
119-
".tex",
120-
".odt",
121-
".odp",
122-
]
123-
124100
def get_inputs(self):
125101
"""Locate the ``from`` and ``to`` input files."""
126102
self.from_files, self.to_files = d2d.get_inputs(self.project)
@@ -155,6 +131,15 @@ def flag_whitespace_files(self):
155131
"""Flag whitespace files with size less than or equal to 100 byte as ignored."""
156132
d2d.flag_whitespace_files(project=self.project)
157133

134+
def load_ecosystem_config(self):
135+
"""Load ecosystem specific configurations for d2d steps for selected options."""
136+
d2d_config.load_ecosystem_config(pipeline=self, options=self.selected_groups)
137+
138+
@optional_step("Ruby")
139+
def map_ruby(self):
140+
"""Load Ruby specific configurations for d2d steps."""
141+
pass
142+
158143
def map_about_files(self):
159144
"""Map ``from/`` .ABOUT files to their related ``to/`` resources."""
160145
d2d.map_about_files(project=self.project, logger=self.log)
@@ -171,7 +156,7 @@ def match_archives_to_purldb(self):
171156

172157
d2d.match_purldb_resources(
173158
project=self.project,
174-
extensions=self.purldb_package_extensions,
159+
extensions=self.matchable_package_extensions,
175160
matcher_func=d2d.match_purldb_package,
176161
logger=self.log,
177162
)
@@ -249,7 +234,7 @@ def match_resources_to_purldb(self):
249234

250235
d2d.match_purldb_resources(
251236
project=self.project,
252-
extensions=self.purldb_resource_extensions,
237+
extensions=self.matchable_resource_extensions,
253238
matcher_func=d2d.match_purldb_resource,
254239
logger=self.log,
255240
)
@@ -287,6 +272,7 @@ def flag_mapped_resources_archives_and_ignored_directories(self):
287272
def perform_house_keeping_tasks(self):
288273
"""
289274
On deployed side
275+
- Ignore specific files based on ecosystem based configurations.
290276
- PurlDB match files with ``no-java-source`` and empty status,
291277
if no match is found update status to ``requires-review``.
292278
- Update status for uninteresting files.
@@ -297,9 +283,14 @@ def perform_house_keeping_tasks(self):
297283
"""
298284
d2d.match_resources_with_no_java_source(project=self.project, logger=self.log)
299285
d2d.handle_dangling_deployed_legal_files(project=self.project, logger=self.log)
286+
d2d.ignore_unmapped_resources_from_config(
287+
project=self.project,
288+
patterns_to_ignore=self.ecosystem_config.deployed_resource_path_exclusions,
289+
logger=self.log,
290+
)
300291
d2d.match_unmapped_resources(
301292
project=self.project,
302-
matched_extensions=self.purldb_resource_extensions,
293+
matched_extensions=self.ecosystem_config.matchable_resource_extensions,
303294
logger=self.log,
304295
)
305296
d2d.flag_undeployed_resources(project=self.project)
@@ -335,5 +326,5 @@ def flag_deployed_from_resources_with_missing_license(self):
335326
"""Update the status for deployed from files with missing license."""
336327
d2d.flag_deployed_from_resources_with_missing_license(
337328
self.project,
338-
doc_extensions=self.doc_extensions,
329+
doc_extensions=self.ecosystem_config.doc_extensions,
339330
)

scanpipe/pipes/d2d.py

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
from scanpipe.models import CodebaseRelation
5555
from scanpipe.models import CodebaseResource
5656
from scanpipe.models import convert_glob_to_django_regex
57+
from scanpipe.pipes import d2d_config
5758
from scanpipe.pipes import flag
5859
from scanpipe.pipes import get_resource_diff_ratio
5960
from scanpipe.pipes import js
@@ -1463,6 +1464,20 @@ def match_resources_with_no_java_source(project, logger=None):
14631464
)
14641465

14651466

1467+
def ignore_unmapped_resources_from_config(project, patterns_to_ignore, logger=None):
1468+
"""Ignore unmapped resources for a project using `patterns_to_ignore`."""
1469+
ignored_resources_count = flag.flag_ignored_patterns(
1470+
codebaseresources=project.codebaseresources.to_codebase().no_status(),
1471+
patterns=patterns_to_ignore,
1472+
status=flag.IGNORED_FROM_CONFIG,
1473+
)
1474+
if logger:
1475+
logger(
1476+
f"Ignoring {ignored_resources_count:,d} to/ resources with "
1477+
"ecosystem specific configurations."
1478+
)
1479+
1480+
14661481
def match_unmapped_resources(project, matched_extensions=None, logger=None):
14671482
"""
14681483
Match resources with empty status to PurlDB, if unmatched
@@ -1925,7 +1940,10 @@ def map_rust_binaries_with_symbols(project, logger=None):
19251940
)
19261941

19271942
# Collect source symbols from rust source files
1928-
rust_from_resources = from_resources.filter(extension=".rs")
1943+
rust_config = d2d_config.get_ecosystem_config(ecosystem="Rust")
1944+
rust_from_resources = from_resources.filter(
1945+
extension__in=rust_config.source_symbol_extensions
1946+
)
19291947

19301948
map_binaries_with_symbols(
19311949
project=project,
@@ -1945,7 +1963,10 @@ def map_elfs_binaries_with_symbols(project, logger=None):
19451963
)
19461964

19471965
# Collect source symbols from elf related source files
1948-
elf_from_resources = from_resources.filter(extension__in=[".c", ".cpp", ".h"])
1966+
elf_config = d2d_config.get_ecosystem_config(ecosystem="Elf")
1967+
elf_from_resources = from_resources.filter(
1968+
extension__in=elf_config.source_symbol_extensions
1969+
)
19491970

19501971
map_binaries_with_symbols(
19511972
project=project,
@@ -1968,8 +1989,9 @@ def map_macho_binaries_with_symbols(project, logger=None):
19681989
)
19691990

19701991
# Collect source symbols from macos related source files
1992+
macos_config = d2d_config.get_ecosystem_config(ecosystem="MacOS")
19711993
mac_from_resources = from_resources.filter(
1972-
extension__in=[".c", ".cpp", ".h", ".m", ".swift"]
1994+
extension__in=macos_config.source_symbol_extensions,
19731995
)
19741996

19751997
map_binaries_with_symbols(
@@ -1990,8 +2012,9 @@ def map_winpe_binaries_with_symbols(project, logger=None):
19902012
)
19912013

19922014
# Collect source symbols from windows related source files
2015+
windows_config = d2d_config.get_ecosystem_config(ecosystem="Windows")
19932016
windows_from_resources = from_resources.filter(
1994-
extension__in=[".c", ".cpp", ".h", ".cs"]
2017+
extension__in=windows_config.source_symbol_extensions,
19952018
)
19962019

19972020
map_binaries_with_symbols(
@@ -2057,16 +2080,17 @@ def map_javascript_symbols(project, logger=None):
20572080
"""Map deployed JavaScript, TypeScript to its sources using symbols."""
20582081
project_files = project.codebaseresources.files()
20592082

2083+
js_config = d2d_config.get_ecosystem_config(ecosystem="JavaScript")
20602084
javascript_to_resources = (
20612085
project_files.to_codebase()
20622086
.has_no_relation()
2063-
.filter(extension__in=[".ts", ".js"])
2087+
.filter(extension__in=js_config.source_symbol_extensions)
20642088
)
20652089

20662090
javascript_from_resources = (
20672091
project_files.from_codebase()
20682092
.exclude(path__contains="/test/")
2069-
.filter(extension__in=[".ts", ".js"])
2093+
.filter(extension__in=js_config.source_symbol_extensions)
20702094
)
20712095

20722096
if not (javascript_from_resources.exists() and javascript_to_resources.exists()):

0 commit comments

Comments
 (0)