From 6f1cc0b4d548bf905276d4d7fc1e8a4b8209cb5c Mon Sep 17 00:00:00 2001 From: rejzzzz Date: Wed, 24 Dec 2025 18:04:34 +0530 Subject: [PATCH 1/2] feat(d2d): enable mapping multiple binaries against single source #1949 Signed-off-by: rejzzzz --- scanpipe/pipelines/deploy_to_develop.py | 160 +++++----- scanpipe/pipes/d2d.py | 385 +++++++++++++++--------- 2 files changed, 330 insertions(+), 215 deletions(-) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 9d60ac1dc4..e51ec7e371 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -120,22 +120,49 @@ def get_inputs(self): def extract_inputs_to_codebase_directory(self): """Extract input files to the project's codebase/ directory.""" - inputs_with_codebase_path_destination = [ - (self.from_files, self.project.codebase_path / d2d.FROM), - (self.to_files, self.project.codebase_path / d2d.TO), - ] - - for input_files, codebase_path in inputs_with_codebase_path_destination: - for input_file_path in input_files: - if input.is_archive(input_file_path): - self.extract_archive(input_file_path, codebase_path) - else: - input.copy_input(input_file_path, codebase_path) + # Extract from files to FROM + from_path = self.project.codebase_path / d2d.FROM + for input_file_path in self.from_files: + if input.is_archive(input_file_path): + self.extract_archive(input_file_path, from_path) + else: + input.copy_input(input_file_path, from_path) + + # Extract each to file to a separate subdirectory under TO + self.to_paths = [] + for i, input_file_path in enumerate(self.to_files): + if len(self.to_files) > 1: + to_subpath = self.project.codebase_path / d2d.TO / str(i) + else: + to_subpath = self.project.codebase_path / d2d.TO + + self.to_paths.append( + str(to_subpath.relative_to(self.project.codebase_path)) + "/" + ) + + if input.is_archive(input_file_path): + self.extract_archive(input_file_path, to_subpath) + else: + input.copy_input(input_file_path, to_subpath) # Reload the project env post-extraction as the scancode-config.yml file # may be located in one of the extracted archives. self.env = self.project.get_env() + def run_d2d_step(self, func, *args, **kwargs): + """Run a d2d mapping step for each to_path.""" + to_paths = getattr(self, "to_paths", [None]) + for to_path in to_paths: + if to_path: + kwargs["to_queryset"] = self.project.codebaseresources.filter( + path__startswith=to_path + ) + kwargs["to_path"] = to_path + if len(to_paths) > 1: + self.log(f"Running {func.__name__} for {to_path}") + + func(project=self.project, logger=self.log, *args, **kwargs) + def collect_and_create_codebase_resources(self): """Collect and create codebase resources.""" pipes.collect_and_create_codebase_resources(self.project) @@ -159,11 +186,11 @@ def map_ruby(self): def map_about_files(self): """Map ``from/`` .ABOUT files to their related ``to/`` resources.""" - d2d.map_about_files(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_about_files) def map_checksum(self): """Map using SHA1 checksum.""" - d2d.map_checksum(project=self.project, checksum_field="sha1", logger=self.log) + self.run_d2d_step(d2d.map_checksum, checksum_field="sha1") def match_archives_to_purldb(self): """Match selected package archives by extension to PurlDB.""" @@ -171,11 +198,10 @@ def match_archives_to_purldb(self): self.log("PurlDB is not available. Skipping.") return - d2d.match_purldb_resources( - project=self.project, + self.run_d2d_step( + d2d.match_purldb_resources, extensions=self.ecosystem_config.matchable_package_extensions, matcher_func=d2d.match_purldb_package, - logger=self.log, ) @optional_step("Java") @@ -188,16 +214,12 @@ def find_java_packages(self): @optional_step("Java") def map_java_to_class(self): """Map a .class compiled file to its .java source.""" - d2d.map_jvm_to_class( - project=self.project, jvm_lang=jvm.JavaLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.JavaLanguage) @optional_step("Java") def map_jar_to_java_source(self): """Map .jar files to their related source directory.""" - d2d.map_jar_to_jvm_source( - project=self.project, jvm_lang=jvm.JavaLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jar_to_jvm_source, jvm_lang=jvm.JavaLanguage) @optional_step("Scala") def find_scala_packages(self): @@ -209,16 +231,12 @@ def find_scala_packages(self): @optional_step("Scala") def map_scala_to_class(self): """Map a .class compiled file to its .java source.""" - d2d.map_jvm_to_class( - project=self.project, jvm_lang=jvm.ScalaLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.ScalaLanguage) @optional_step("Scala") def map_jar_to_scala_source(self): """Map .jar files to their related source directory.""" - d2d.map_jar_to_jvm_source( - project=self.project, jvm_lang=jvm.ScalaLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jar_to_jvm_source, jvm_lang=jvm.ScalaLanguage) @optional_step("Kotlin") def find_kotlin_packages(self): @@ -230,16 +248,12 @@ def find_kotlin_packages(self): @optional_step("Kotlin") def map_kotlin_to_class(self): """Map a .class compiled file to its .java source.""" - d2d.map_jvm_to_class( - project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.KotlinLanguage) @optional_step("Kotlin") def map_jar_to_kotlin_source(self): """Map .jar files to their related source directory.""" - d2d.map_jar_to_jvm_source( - project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jar_to_jvm_source, jvm_lang=jvm.KotlinLanguage) @optional_step("Grammar") def find_grammar_packages(self): @@ -251,16 +265,12 @@ def find_grammar_packages(self): @optional_step("Grammar") def map_grammar_to_class(self): """Map a .class compiled file to its .g/.g4 source.""" - d2d.map_jvm_to_class( - project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.GrammarLanguage) @optional_step("Grammar") def map_jar_to_grammar_source(self): """Map .jar files to their related source directory.""" - d2d.map_jar_to_jvm_source( - project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jar_to_jvm_source, jvm_lang=jvm.GrammarLanguage) @optional_step("Xtend") def find_xtend_packages(self): @@ -272,9 +282,7 @@ def find_xtend_packages(self): @optional_step("Xtend") def map_xtend_to_class(self): """Map a .class compiled file to its xtend source.""" - d2d.map_jvm_to_class( - project=self.project, jvm_lang=jvm.XtendLanguage, logger=self.log - ) + self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.XtendLanguage) @optional_step("JavaScript") def map_javascript(self): @@ -282,52 +290,48 @@ def map_javascript(self): Map a packed or minified JavaScript, TypeScript, CSS and SCSS to its source. """ - d2d.map_javascript(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_javascript) @optional_step("JavaScript") def map_javascript_symbols(self): """Map deployed JavaScript, TypeScript to its sources using symbols.""" - d2d.map_javascript_symbols(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_javascript_symbols) @optional_step("JavaScript") def map_javascript_strings(self): """Map deployed JavaScript, TypeScript to its sources using string literals.""" - d2d.map_javascript_strings(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_javascript_strings) def get_symbols_from_binaries(self): """Extract symbols from Elf, Mach0 and windows binaries for mapping.""" - d2d.extract_binary_symbols( - project=self.project, - options=self.selected_groups, - logger=self.log, - ) + self.run_d2d_step(d2d.extract_binary_symbols, options=self.selected_groups) @optional_step("Elf") def map_elf(self): """Map ELF binaries to their sources using dwarf paths and symbols.""" - d2d.map_elfs_with_dwarf_paths(project=self.project, logger=self.log) - d2d.map_elfs_binaries_with_symbols(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_elfs_with_dwarf_paths) + self.run_d2d_step(d2d.map_elfs_binaries_with_symbols) @optional_step("MacOS") def map_macho(self): """Map mach0 binaries to their sources using symbols.""" - d2d.map_macho_binaries_with_symbols(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_macho_binaries_with_symbols) @optional_step("Windows") def map_winpe(self): """Map winpe binaries to their sources using symbols.""" - d2d.map_winpe_binaries_with_symbols(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_winpe_binaries_with_symbols) @optional_step("Go") def map_go(self): """Map Go binaries to their sources using paths and symbols.""" - d2d.map_go_paths(project=self.project, logger=self.log) - d2d.map_go_binaries_with_symbols(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_go_paths) + self.run_d2d_step(d2d.map_go_binaries_with_symbols) @optional_step("Rust") def map_rust(self): """Map Rust binaries to their sources using symbols.""" - d2d.map_rust_binaries_with_symbols(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_rust_binaries_with_symbols) @optional_step("Python") def map_python(self): @@ -335,8 +339,8 @@ def map_python(self): Map binaries from Python packages to their sources using dwarf paths and symbols. """ - d2d.map_python_pyx_to_binaries(project=self.project, logger=self.log) - d2d.map_python_protobuf_files(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_python_pyx_to_binaries) + self.run_d2d_step(d2d.map_python_protobuf_files) def match_directories_to_purldb(self): """Match selected directories in PurlDB.""" @@ -344,10 +348,7 @@ def match_directories_to_purldb(self): self.log("PurlDB is not available. Skipping.") return - d2d.match_purldb_directories( - project=self.project, - logger=self.log, - ) + self.run_d2d_step(d2d.match_purldb_directories) def match_resources_to_purldb(self): """Match selected files by extension in PurlDB.""" @@ -355,42 +356,41 @@ def match_resources_to_purldb(self): self.log("PurlDB is not available. Skipping.") return - d2d.match_purldb_resources( - project=self.project, + self.run_d2d_step( + d2d.match_purldb_resources, extensions=self.ecosystem_config.matchable_resource_extensions, matcher_func=d2d.match_purldb_resource, - logger=self.log, ) @optional_step("JavaScript") def map_javascript_post_purldb_match(self): """Map minified javascript file based on existing PurlDB match.""" - d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_javascript_post_purldb_match) @optional_step("JavaScript") def map_javascript_path(self): """Map javascript file based on path.""" - d2d.map_javascript_path(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_javascript_path) @optional_step("JavaScript") def map_javascript_colocation(self): """Map JavaScript files based on neighborhood file mapping.""" - d2d.map_javascript_colocation(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_javascript_colocation) @optional_step("JavaScript") def map_thirdparty_npm_packages(self): """Map thirdparty package using package.json metadata.""" - d2d.map_thirdparty_npm_packages(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_thirdparty_npm_packages) def map_path(self): """Map using path similarities.""" - d2d.map_path(project=self.project, logger=self.log) + self.run_d2d_step(d2d.map_path) def flag_mapped_resources_archives_and_ignored_directories(self): """Flag all codebase resources that were mapped during the pipeline.""" flag.flag_mapped_resources(self.project) flag.flag_ignored_directories(self.project) - d2d.flag_processed_archives(self.project) + self.run_d2d_step(d2d.flag_processed_archives) def perform_house_keeping_tasks(self): """ @@ -404,17 +404,15 @@ def perform_house_keeping_tasks(self): On devel side - Update status for not deployed files. """ - d2d.match_resources_with_no_java_source(project=self.project, logger=self.log) - d2d.handle_dangling_deployed_legal_files(project=self.project, logger=self.log) - d2d.ignore_unmapped_resources_from_config( - project=self.project, + self.run_d2d_step(d2d.match_resources_with_no_java_source) + self.run_d2d_step(d2d.handle_dangling_deployed_legal_files) + self.run_d2d_step( + d2d.ignore_unmapped_resources_from_config, patterns_to_ignore=self.ecosystem_config.deployed_resource_path_exclusions, - logger=self.log, ) - d2d.match_unmapped_resources( - project=self.project, + self.run_d2d_step( + d2d.match_unmapped_resources, matched_extensions=self.ecosystem_config.matchable_resource_extensions, - logger=self.log, ) d2d.flag_undeployed_resources(project=self.project) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 5046a61b09..deec2941d0 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -139,13 +139,17 @@ def _map_checksum_resource(to_resource, from_resources, checksum_field): ) -def map_checksum(project, checksum_field, logger=None): +def map_checksum(project, checksum_field, logger=None, to_queryset=None): """Map using checksum.""" project_files = project.codebaseresources.files().no_status() from_resources = project_files.from_codebase().has_value(checksum_field) - to_resources = ( - project_files.to_codebase().has_value(checksum_field).has_no_relation() - ) + + if to_queryset is not None: + to_resources = to_queryset.files().no_status() + else: + to_resources = project_files.to_codebase() + + to_resources = to_resources.has_value(checksum_field).has_no_relation() resource_count = to_resources.count() if logger: @@ -192,7 +196,7 @@ def _map_jvm_to_class_resource( ) -def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None): +def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None, to_queryset=None): """ Map to/ compiled Jvm's binary files to from/ using Jvm language's fully qualified paths and indexing from/ Jvm lang's source files. @@ -205,7 +209,11 @@ def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None): # .class files failing to resolve. See # https://github.com/aboutcode-org/scancode.io/issues/1854#issuecomment-3273472895 from_resources = project_files.from_codebase() - to_resources = project_files.to_codebase().no_status().has_no_relation() + + if to_queryset is not None: + to_resources = to_queryset.files().no_status().has_no_relation() + else: + to_resources = project_files.to_codebase().no_status().has_no_relation() has_source_pkg_attr_name = { f"extra_data__{jvm_lang.source_package_attribute_name}__isnull": False @@ -340,12 +348,17 @@ def _map_jar_to_jvm_source_resource( ) -def map_jar_to_jvm_source(project, jvm_lang: jvm.JvmLanguage, logger=None): +def map_jar_to_jvm_source(project, jvm_lang: jvm.JvmLanguage, logger=None, to_queryset=None): """Map .jar files to their related source directory.""" project_files = project.codebaseresources.files() # Include the directories to map on the common source from_resources = project.codebaseresources.from_codebase() - to_resources = project_files.to_codebase() + + if to_queryset is not None: + to_resources = to_queryset.files() + else: + to_resources = project_files.to_codebase() + to_jars = to_resources.filter(extension=".jar") to_jars_count = to_jars.count() @@ -402,11 +415,16 @@ def _map_path_resource( ) -def map_path(project, logger=None): +def map_path(project, logger=None, to_queryset=None): """Map using path suffix similarities.""" project_files = project.codebaseresources.files().no_status() from_resources = project_files.from_codebase() - to_resources = project_files.to_codebase().has_no_relation() + + if to_queryset is not None: + to_resources = to_queryset.files().no_status().has_no_relation() + else: + to_resources = project_files.to_codebase().has_no_relation() + resource_count = to_resources.count() if logger: @@ -602,7 +620,7 @@ def match_sha1s_to_purldb( def match_purldb_resources( - project, extensions, matcher_func, chunk_size=1000, logger=None + project, extensions, matcher_func, chunk_size=1000, logger=None, to_queryset=None ): """ Match against PurlDB selecting codebase resources using provided @@ -611,11 +629,13 @@ def match_purldb_resources( Match requests are sent off in batches of 1000 SHA1s. This number is set using `chunk_size`. """ + if to_queryset is not None: + to_resources = to_queryset.files().no_status() + else: + to_resources = project.codebaseresources.files().to_codebase().no_status() + to_resources = ( - project.codebaseresources.files() - .to_codebase() - .no_status() - .has_value("sha1") + to_resources.has_value("sha1") .filter(extension__in=extensions) ) resource_count = to_resources.count() @@ -690,7 +710,7 @@ def _match_purldb_resources( ) -def match_purldb_directories(project, logger=None): +def match_purldb_directories(project, logger=None, to_queryset=None): """Match against PurlDB selecting codebase directories.""" # If we are able to get match results for a directory fingerprint, then that # means every resource and directory under that directory is part of a @@ -698,10 +718,13 @@ def match_purldb_directories(project, logger=None): # match as many files as we can before attempting to match further down. The # more "higher-up" directories we can match to means that we reduce the # number of queries made to purldb. + if to_queryset is not None: + to_directories = to_queryset.directories() + else: + to_directories = project.codebaseresources.directories().to_codebase() + to_directories = ( - project.codebaseresources.directories() - .to_codebase() - .no_status(status=flag.ABOUT_MAPPED) + to_directories.no_status(status=flag.ABOUT_MAPPED) .no_status(status=flag.MATCHED_TO_PURLDB_PACKAGE) .order_by("path") ) @@ -721,25 +744,31 @@ def match_purldb_directories(project, logger=None): if directory.status != flag.MATCHED_TO_PURLDB_DIRECTORY: match_purldb_directory(project, directory) - matched_count = ( - project.codebaseresources.directories() - .to_codebase() - .filter(status=flag.MATCHED_TO_PURLDB_DIRECTORY) - .count() - ) + if to_queryset is not None: + matched_directories = to_queryset.directories() + else: + matched_directories = project.codebaseresources.directories().to_codebase() + + matched_count = matched_directories.filter( + status=flag.MATCHED_TO_PURLDB_DIRECTORY + ).count() logger( f"{matched_count:,d} director{pluralize(matched_count, 'y,ies')} " f"matched in PurlDB" ) -def map_javascript(project, logger=None): +def map_javascript(project, logger=None, to_queryset=None): """Map a packed or minified JavaScript, TypeScript, CSS and SCSS to its source.""" project_files = project.codebaseresources.files() - to_resources = project_files.to_codebase().no_status().exclude(name__startswith=".") - to_resources_dot_map = to_resources.filter(extension=".map") - to_resources_minified = to_resources.filter(extension__in=[".css", ".js"]) + if to_queryset is not None: + to_resources_base = to_queryset.files().no_status().exclude(name__startswith=".") + else: + to_resources_base = project_files.to_codebase().no_status().exclude(name__startswith=".") + + to_resources_dot_map = to_resources_base.filter(extension=".map") + to_resources_minified = to_resources_base.filter(extension__in=[".css", ".js"]) to_resources_dot_map_count = to_resources_dot_map.count() if logger: @@ -1006,7 +1035,7 @@ def create_about_packages_relations(self, project): return about_purls, mapped_about_resources -def map_about_files(project, logger=None): +def map_about_files(project, logger=None, to_queryset=None): """Map ``from/`` .ABOUT files to their related ``to/`` resources.""" project_resources = project.codebaseresources from_about_files = ( @@ -1026,7 +1055,11 @@ def map_about_files(project, logger=None): ) # Ignoring empty or ignored files as they are not relevant anyway - to_resources = project_resources.to_codebase().no_status() + if to_queryset is not None: + to_resources = to_queryset.no_status() + else: + to_resources = project_resources.to_codebase().no_status() + mapped_to_resources = indexes.map_deployed_to_devel_using_about( to_resources=to_resources, ) @@ -1047,11 +1080,14 @@ def map_about_files(project, logger=None): ) -def map_javascript_post_purldb_match(project, logger=None): +def map_javascript_post_purldb_match(project, logger=None, to_queryset=None): """Map minified javascript file based on existing PurlDB match.""" project_files = project.codebaseresources.files() - to_resources = project_files.to_codebase() + if to_queryset is not None: + to_resources = to_queryset.files() + else: + to_resources = project_files.to_codebase() to_resources_dot_map = to_resources.filter( status=flag.MATCHED_TO_PURLDB_RESOURCE @@ -1113,19 +1149,23 @@ def _map_javascript_post_purldb_match_resource( to_minified.update(status=flag.MAPPED) -def map_javascript_path(project, logger=None): +def map_javascript_path(project, logger=None, to_queryset=None): """Map javascript file based on path.""" project_files = project.codebaseresources.files() + if to_queryset is not None: + to_resources_base = to_queryset.files().no_status() + else: + to_resources_base = project_files.to_codebase().no_status() + to_resources_key = ( - project_files.to_codebase() - .no_status() + to_resources_base .filter(extension__in=[".map", ".ts"]) .exclude(name__startswith=".") .exclude(path__contains="/node_modules/") ) - to_resources = project_files.to_codebase().no_status().exclude(name__startswith=".") + to_resources = to_resources_base.exclude(name__startswith=".") from_resources = project_files.from_codebase().exclude(path__contains="/test/") resource_count = to_resources_key.count() @@ -1201,19 +1241,23 @@ def _map_javascript_path_resource( ) -def map_javascript_colocation(project, logger=None): +def map_javascript_colocation(project, logger=None, to_queryset=None): """Map JavaScript files based on neighborhood file mapping.""" project_files = project.codebaseresources.files() + if to_queryset is not None: + to_resources_base = to_queryset.files().no_status() + else: + to_resources_base = project_files.to_codebase().no_status() + to_resources_key = ( - project_files.to_codebase() - .no_status() + to_resources_base .filter(extension__in=[".map", ".ts"]) .exclude(name__startswith=".") .exclude(path__contains="/node_modules/") ) - to_resources = project_files.to_codebase().no_status().exclude(name__startswith=".") + to_resources = to_resources_base.exclude(name__startswith=".") from_resources = project_files.from_codebase().exclude(path__contains="/test/") resource_count = to_resources_key.count() @@ -1288,7 +1332,7 @@ def _map_javascript_colocation_resource( ) -def flag_processed_archives(project): +def flag_processed_archives(project, to_queryset=None): """ Flag package archives as processed if they meet the following criteria: @@ -1302,9 +1346,12 @@ def flag_processed_archives(project): have statuses. If so, it updates the status of the package archive to "archive-processed". """ - to_resources = ( - project.codebaseresources.all().to_codebase().no_status().order_by("-path") - ) + if to_queryset is not None: + to_resources = to_queryset.no_status().order_by("-path") + else: + to_resources = ( + project.codebaseresources.all().to_codebase().no_status().order_by("-path") + ) for archive_resource in to_resources.archives(): extract_path = archive_resource.path + EXTRACT_SUFFIX @@ -1315,17 +1362,22 @@ def flag_processed_archives(project): archive_resource.update(status=flag.ARCHIVE_PROCESSED) -def map_thirdparty_npm_packages(project, logger=None): +def map_thirdparty_npm_packages(project, logger=None, to_queryset=None): """Map thirdparty package using package.json metadata.""" project_files = project.codebaseresources.files() + if to_queryset is not None: + to_resources_base = to_queryset.files() + else: + to_resources_base = project_files.to_codebase() + to_package_json = ( - project_files.to_codebase() + to_resources_base .filter(path__regex=r"^.*\/node_modules\/.*\/package\.json$") .exclude(path__regex=r"^.*\/node_modules\/.*\/node_modules\/.*$") ) - to_resources = project_files.to_codebase().no_status() + to_resources = to_resources_base.no_status() resource_count = to_package_json.count() if logger: @@ -1421,14 +1473,17 @@ def create_local_files_packages(project): pipes.create_local_files_package(project, defaults, codebase_resource_ids) -def match_resources_with_no_java_source(project, logger=None): +def match_resources_with_no_java_source(project, logger=None, to_queryset=None): """ Match resources with ``no-java-source`` to PurlDB, if no match is found update status to ``requires-review``. """ project_files = project.codebaseresources.files() - to_no_java_source = project_files.to_codebase().filter(status=flag.NO_JAVA_SOURCE) + if to_queryset is not None: + to_no_java_source = to_queryset.files().filter(status=flag.NO_JAVA_SOURCE) + else: + to_no_java_source = project_files.to_codebase().filter(status=flag.NO_JAVA_SOURCE) if to_no_java_source: resource_count = to_no_java_source.count() @@ -1449,10 +1504,17 @@ def match_resources_with_no_java_source(project, logger=None): ) -def ignore_unmapped_resources_from_config(project, patterns_to_ignore, logger=None): +def ignore_unmapped_resources_from_config( + project, patterns_to_ignore, logger=None, to_queryset=None +): """Ignore unmapped resources for a project using `patterns_to_ignore`.""" + if to_queryset is not None: + to_resources = to_queryset.no_status() + else: + to_resources = project.codebaseresources.to_codebase().no_status() + ignored_resources_count = flag.flag_ignored_patterns( - codebaseresources=project.codebaseresources.to_codebase().no_status(), + codebaseresources=to_resources, patterns=patterns_to_ignore, status=flag.IGNORED_FROM_CONFIG, ) @@ -1463,14 +1525,21 @@ def ignore_unmapped_resources_from_config(project, patterns_to_ignore, logger=No ) -def match_unmapped_resources(project, matched_extensions=None, logger=None): +def match_unmapped_resources( + project, matched_extensions=None, logger=None, to_queryset=None +): """ Match resources with empty status to PurlDB, if unmatched update status as ``requires-review``. """ project_files = project.codebaseresources.files() - to_unmapped = project_files.to_codebase().no_status().exclude(is_media=True) + if to_queryset is not None: + to_unmapped = to_queryset.files().no_status() + else: + to_unmapped = project_files.to_codebase().no_status() + + to_unmapped = to_unmapped.exclude(is_media=True) if matched_extensions: to_unmapped.exclude(extension__in=matched_extensions) @@ -1507,36 +1576,38 @@ def flag_undeployed_resources(project): from_unmapped.update(status=flag.NOT_DEPLOYED) -def scan_ignored_to_files(project, logger=None): +def scan_ignored_to_files(project, logger=None, to_queryset=None): """ Scan status="ignored-from-config" ``to/`` files for copyrights, licenses, emails, and urls. """ - scan_files = ( - project.codebaseresources.files() - .to_codebase() - .filter(status=flag.IGNORED_FROM_CONFIG) - ) + if to_queryset is not None: + to_resources = to_queryset.files() + else: + to_resources = project.codebaseresources.files().to_codebase() + + scan_files = to_resources.filter(status=flag.IGNORED_FROM_CONFIG) scancode.scan_for_files(project, scan_files, progress_logger=logger) - project.codebaseresources.files().to_codebase().filter(status=flag.SCANNED).update( + to_resources.filter(status=flag.SCANNED).update( status=flag.IGNORED_FROM_CONFIG ) -def scan_unmapped_to_files(project, logger=None): +def scan_unmapped_to_files(project, logger=None, to_queryset=None): """ Scan unmapped/matched ``to/`` files for copyrights, licenses, emails, and urls and update the status to `requires-review`. """ - scan_files = ( - project.codebaseresources.files() - .to_codebase() - .filter(status=flag.REQUIRES_REVIEW) - ) + if to_queryset is not None: + to_resources = to_queryset.files() + else: + to_resources = project.codebaseresources.files().to_codebase() + + scan_files = to_resources.filter(status=flag.REQUIRES_REVIEW) scancode.scan_for_files(project, scan_files, progress_logger=logger) - project.codebaseresources.files().to_codebase().filter(status=flag.SCANNED).update( + to_resources.filter(status=flag.SCANNED).update( status=flag.REQUIRES_REVIEW ) @@ -1565,12 +1636,15 @@ def flag_deployed_from_resources_with_missing_license(project, doc_extensions=No unknown_license_files.update(status=flag.UNKNOWN_LICENSE) -def handle_dangling_deployed_legal_files(project, logger): +def handle_dangling_deployed_legal_files(project, logger, to_queryset=None): """ Scan the legal files with empty status and update status to `REVIEW_DANGLING_LEGAL_FILE`. """ - to_resources = project.codebaseresources.files().to_codebase().no_status() + if to_queryset is not None: + to_resources = to_queryset.files().no_status() + else: + to_resources = project.codebaseresources.files().to_codebase().no_status() legal_file_filter = Q() @@ -1626,15 +1700,18 @@ def flag_whitespace_files(project): resource.update(status=flag.IGNORED_WHITESPACE_FILE) -def match_purldb_resources_post_process(project, logger=None): +def match_purldb_resources_post_process(project, logger=None, to_queryset=None): """Choose the best package for PurlDB matched resources.""" - to_extract_directories = ( - project.codebaseresources.directories() - .to_codebase() - .filter(path__regex=r"^.*-extract$") - ) + if to_queryset is not None: + to_extract_directories = to_queryset.directories() + to_resources_base = to_queryset.files() + else: + to_extract_directories = project.codebaseresources.directories().to_codebase() + to_resources_base = project.codebaseresources.files().to_codebase() - to_resources = project.codebaseresources.files().filter( + to_extract_directories = to_extract_directories.filter(path__regex=r"^.*-extract$") + + to_resources = to_resources_base.filter( status=flag.MATCHED_TO_PURLDB_RESOURCE ) @@ -1815,12 +1892,15 @@ def is_invalid_match(match, matched_path_length): return matched_path_length == 1 and len(match.resource_ids) != 1 -def map_elfs_with_dwarf_paths(project, logger=None): +def map_elfs_with_dwarf_paths(project, logger=None, to_queryset=None): """Map ELF binaries to their sources in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - to_resources = ( - project.codebaseresources.files().to_codebase().has_no_relation().elfs() - ) + if to_queryset is not None: + to_resources = to_queryset.files().has_no_relation().elfs() + else: + to_resources = ( + project.codebaseresources.files().to_codebase().has_no_relation().elfs() + ) for resource in to_resources: try: paths = get_elf_file_dwarf_paths(resource.location_path) @@ -1884,15 +1964,18 @@ def get_go_file_paths(location): return file_paths -def map_go_paths(project, logger=None): +def map_go_paths(project, logger=None, to_queryset=None): """Map Go binaries to their source in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - to_resources = ( - project.codebaseresources.files() - .to_codebase() - .has_no_relation() - .executable_binaries() - ) + if to_queryset is not None: + to_resources = to_queryset.files().has_no_relation().executable_binaries() + else: + to_resources = ( + project.codebaseresources.files() + .to_codebase() + .has_no_relation() + .executable_binaries() + ) for resource in to_resources: try: paths = get_go_file_paths(resource.location_path) @@ -1937,14 +2020,18 @@ def map_go_paths(project, logger=None): WINPE_BINARY_OPTIONS = ["Windows"] -def extract_binary_symbols(project, options, logger=None): +def extract_binary_symbols(project, options, logger=None, to_queryset=None): """ Extract binary symbols for all Elf, Mach0 and Winpe binaries found in the ``project`` resources, based on selected ecosystem ``options`` so that these symbols can be mapped to extracted source symbols. """ - to_resources = project.codebaseresources.files().to_codebase().has_no_relation() + if to_queryset is not None: + to_resources = to_queryset.files().has_no_relation() + else: + to_resources = project.codebaseresources.files().to_codebase().has_no_relation() + if any([option in ELF_BINARY_OPTIONS for option in options]): to_binaries = to_resources.elfs() extract_binary_symbols_from_resources( @@ -1978,15 +2065,18 @@ def extract_binary_symbols(project, options, logger=None): ) -def map_rust_binaries_with_symbols(project, logger=None): +def map_rust_binaries_with_symbols(project, logger=None, to_queryset=None): """Map Rust binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - to_binaries = ( - project.codebaseresources.files() - .to_codebase() - .has_no_relation() - .executable_binaries() - ) + if to_queryset is not None: + to_binaries = to_queryset.files().has_no_relation().executable_binaries() + else: + to_binaries = ( + project.codebaseresources.files() + .to_codebase() + .has_no_relation() + .executable_binaries() + ) # Collect source symbols from rust source files rust_config = d2d_config.get_ecosystem_config(ecosystem="Rust") @@ -2003,15 +2093,18 @@ def map_rust_binaries_with_symbols(project, logger=None): ) -def map_go_binaries_with_symbols(project, logger=None): +def map_go_binaries_with_symbols(project, logger=None, to_queryset=None): """Map Go binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - to_binaries = ( - project.codebaseresources.files() - .to_codebase() - .has_no_relation() - .executable_binaries() - ) + if to_queryset is not None: + to_binaries = to_queryset.files().has_no_relation().executable_binaries() + else: + to_binaries = ( + project.codebaseresources.files() + .to_codebase() + .has_no_relation() + .executable_binaries() + ) # Collect source symbols from rust source files go_config = d2d_config.get_ecosystem_config(ecosystem="Go") @@ -2028,12 +2121,15 @@ def map_go_binaries_with_symbols(project, logger=None): ) -def map_elfs_binaries_with_symbols(project, logger=None): +def map_elfs_binaries_with_symbols(project, logger=None, to_queryset=None): """Map Elf binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - elf_binaries = ( - project.codebaseresources.files().to_codebase().has_no_relation().elfs() - ) + if to_queryset is not None: + elf_binaries = to_queryset.files().has_no_relation().elfs() + else: + elf_binaries = ( + project.codebaseresources.files().to_codebase().has_no_relation().elfs() + ) # Collect source symbols from elf related source files elf_config = d2d_config.get_ecosystem_config(ecosystem="Elf") @@ -2050,15 +2146,18 @@ def map_elfs_binaries_with_symbols(project, logger=None): ) -def map_macho_binaries_with_symbols(project, logger=None): +def map_macho_binaries_with_symbols(project, logger=None, to_queryset=None): """Map macho binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - macho_binaries = ( - project.codebaseresources.files() - .to_codebase() - .has_no_relation() - .macho_binaries() - ) + if to_queryset is not None: + macho_binaries = to_queryset.files().has_no_relation().macho_binaries() + else: + macho_binaries = ( + project.codebaseresources.files() + .to_codebase() + .has_no_relation() + .macho_binaries() + ) # Collect source symbols from macos related source files macos_config = d2d_config.get_ecosystem_config(ecosystem="MacOS") @@ -2075,12 +2174,15 @@ def map_macho_binaries_with_symbols(project, logger=None): ) -def map_winpe_binaries_with_symbols(project, logger=None): +def map_winpe_binaries_with_symbols(project, logger=None, to_queryset=None): """Map winpe binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - winexe_binaries = ( - project.codebaseresources.files().to_codebase().has_no_relation().win_exes() - ) + if to_queryset is not None: + winexe_binaries = to_queryset.files().has_no_relation().win_exes() + else: + winexe_binaries = ( + project.codebaseresources.files().to_codebase().has_no_relation().win_exes() + ) # Collect source symbols from windows related source files windows_config = d2d_config.get_ecosystem_config(ecosystem="Windows") @@ -2166,13 +2268,18 @@ def extract_binary_symbols_from_resources(resources, binary_symbols_func, logger logger(f"Error parsing binary symbols at: {resource.location_path!r} {e!r}") -def map_javascript_symbols(project, logger=None): +def map_javascript_symbols(project, logger=None, to_queryset=None): """Map deployed JavaScript, TypeScript to its sources using symbols.""" project_files = project.codebaseresources.files() js_config = d2d_config.get_ecosystem_config(ecosystem="JavaScript") + if to_queryset is not None: + to_resources_base = to_queryset.files() + else: + to_resources_base = project_files.to_codebase() + javascript_to_resources = ( - project_files.to_codebase() + to_resources_base .has_no_relation() .filter(extension__in=js_config.source_symbol_extensions) ) @@ -2262,12 +2369,17 @@ def _map_javascript_symbols(to_resource, javascript_from_resources, logger): return 0 -def map_javascript_strings(project, logger=None): +def map_javascript_strings(project, logger=None, to_queryset=None): """Map deployed JavaScript, TypeScript to its sources using string literals.""" project_files = project.codebaseresources.files() + if to_queryset is not None: + to_resources_base = to_queryset.files() + else: + to_resources_base = project_files.to_codebase() + javascript_to_resources = ( - project_files.to_codebase() + to_resources_base .has_no_relation() .filter(extension__in=[".ts", ".js"]) .exclude(extra_data={}) @@ -2345,7 +2457,7 @@ def _map_javascript_strings(to_resource, javascript_from_resources, logger): return 0 -def map_python_pyx_to_binaries(project, logger=None): +def map_python_pyx_to_binaries(project, logger=None, to_queryset=None): """Map Cython source to their compiled binaries in ``project``.""" from source_inspector.symbols_tree_sitter import get_tree_and_language_info @@ -2355,9 +2467,13 @@ def map_python_pyx_to_binaries(project, logger=None): .from_codebase() .filter(extension__in=python_config.source_symbol_extensions) ) - to_resources = ( - project.codebaseresources.files().to_codebase().has_no_relation().elfs() - ) + + if to_queryset is not None: + to_resources = to_queryset.files().has_no_relation().elfs() + else: + to_resources = ( + project.codebaseresources.files().to_codebase().has_no_relation().elfs() + ) for resource in from_resources: # Open Cython source file, create AST, parse it for function definitions @@ -2389,17 +2505,18 @@ def map_python_pyx_to_binaries(project, logger=None): ) -def map_python_protobuf_files(project, logger=None): +def map_python_protobuf_files(project, logger=None, to_queryset=None): """Map protobuf-generated .py/.pyi files to their source .proto files.""" from_resources = ( project.codebaseresources.files().from_codebase().filter(extension=".proto") ) - to_resources = ( - project.codebaseresources.files() - .to_codebase() - .has_no_relation() - .filter(extension__in=[".py", ".pyi"]) - ) + + if to_queryset is not None: + to_resources = to_queryset.files().has_no_relation() + else: + to_resources = project.codebaseresources.files().to_codebase().has_no_relation() + + to_resources = to_resources.filter(extension__in=[".py", ".pyi"]) to_resources_count = to_resources.count() from_resources_count = from_resources.count() From 78cfe014b4fc3534824d2c2d6cf6a3d0ba0e5061 Mon Sep 17 00:00:00 2001 From: rejzzzz Date: Fri, 26 Dec 2025 22:46:12 +0530 Subject: [PATCH 2/2] fix(d2d): add **kwargs to pipe functions for pipeline compatibility Signed-off-by: rejzzzz --- scanpipe/pipes/d2d.py | 66 +++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index deec2941d0..c95ac038cb 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -139,7 +139,7 @@ def _map_checksum_resource(to_resource, from_resources, checksum_field): ) -def map_checksum(project, checksum_field, logger=None, to_queryset=None): +def map_checksum(project, checksum_field, logger=None, to_queryset=None, **kwargs): """Map using checksum.""" project_files = project.codebaseresources.files().no_status() from_resources = project_files.from_codebase().has_value(checksum_field) @@ -196,7 +196,9 @@ def _map_jvm_to_class_resource( ) -def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None, to_queryset=None): +def map_jvm_to_class( + project, jvm_lang: jvm.JvmLanguage, logger=None, to_queryset=None, **kwargs +): """ Map to/ compiled Jvm's binary files to from/ using Jvm language's fully qualified paths and indexing from/ Jvm lang's source files. @@ -348,7 +350,9 @@ def _map_jar_to_jvm_source_resource( ) -def map_jar_to_jvm_source(project, jvm_lang: jvm.JvmLanguage, logger=None, to_queryset=None): +def map_jar_to_jvm_source( + project, jvm_lang: jvm.JvmLanguage, logger=None, to_queryset=None, **kwargs +): """Map .jar files to their related source directory.""" project_files = project.codebaseresources.files() # Include the directories to map on the common source @@ -415,7 +419,7 @@ def _map_path_resource( ) -def map_path(project, logger=None, to_queryset=None): +def map_path(project, logger=None, to_queryset=None, **kwargs): """Map using path suffix similarities.""" project_files = project.codebaseresources.files().no_status() from_resources = project_files.from_codebase() @@ -620,7 +624,13 @@ def match_sha1s_to_purldb( def match_purldb_resources( - project, extensions, matcher_func, chunk_size=1000, logger=None, to_queryset=None + project, + extensions, + matcher_func, + chunk_size=1000, + logger=None, + to_queryset=None, + **kwargs, ): """ Match against PurlDB selecting codebase resources using provided @@ -758,7 +768,7 @@ def match_purldb_directories(project, logger=None, to_queryset=None): ) -def map_javascript(project, logger=None, to_queryset=None): +def map_javascript(project, logger=None, to_queryset=None, **kwargs): """Map a packed or minified JavaScript, TypeScript, CSS and SCSS to its source.""" project_files = project.codebaseresources.files() @@ -1035,7 +1045,7 @@ def create_about_packages_relations(self, project): return about_purls, mapped_about_resources -def map_about_files(project, logger=None, to_queryset=None): +def map_about_files(project, logger=None, to_queryset=None, **kwargs): """Map ``from/`` .ABOUT files to their related ``to/`` resources.""" project_resources = project.codebaseresources from_about_files = ( @@ -1080,7 +1090,7 @@ def map_about_files(project, logger=None, to_queryset=None): ) -def map_javascript_post_purldb_match(project, logger=None, to_queryset=None): +def map_javascript_post_purldb_match(project, logger=None, to_queryset=None, **kwargs): """Map minified javascript file based on existing PurlDB match.""" project_files = project.codebaseresources.files() @@ -1149,7 +1159,7 @@ def _map_javascript_post_purldb_match_resource( to_minified.update(status=flag.MAPPED) -def map_javascript_path(project, logger=None, to_queryset=None): +def map_javascript_path(project, logger=None, to_queryset=None, **kwargs): """Map javascript file based on path.""" project_files = project.codebaseresources.files() @@ -1241,7 +1251,7 @@ def _map_javascript_path_resource( ) -def map_javascript_colocation(project, logger=None, to_queryset=None): +def map_javascript_colocation(project, logger=None, to_queryset=None, **kwargs): """Map JavaScript files based on neighborhood file mapping.""" project_files = project.codebaseresources.files() @@ -1332,7 +1342,7 @@ def _map_javascript_colocation_resource( ) -def flag_processed_archives(project, to_queryset=None): +def flag_processed_archives(project, logger=None, to_queryset=None, **kwargs): """ Flag package archives as processed if they meet the following criteria: @@ -1362,7 +1372,7 @@ def flag_processed_archives(project, to_queryset=None): archive_resource.update(status=flag.ARCHIVE_PROCESSED) -def map_thirdparty_npm_packages(project, logger=None, to_queryset=None): +def map_thirdparty_npm_packages(project, logger=None, to_queryset=None, **kwargs): """Map thirdparty package using package.json metadata.""" project_files = project.codebaseresources.files() @@ -1473,7 +1483,7 @@ def create_local_files_packages(project): pipes.create_local_files_package(project, defaults, codebase_resource_ids) -def match_resources_with_no_java_source(project, logger=None, to_queryset=None): +def match_resources_with_no_java_source(project, logger=None, to_queryset=None, **kwargs): """ Match resources with ``no-java-source`` to PurlDB, if no match is found update status to ``requires-review``. @@ -1505,7 +1515,7 @@ def match_resources_with_no_java_source(project, logger=None, to_queryset=None): def ignore_unmapped_resources_from_config( - project, patterns_to_ignore, logger=None, to_queryset=None + project, patterns_to_ignore, logger=None, to_queryset=None, **kwargs ): """Ignore unmapped resources for a project using `patterns_to_ignore`.""" if to_queryset is not None: @@ -1526,7 +1536,7 @@ def ignore_unmapped_resources_from_config( def match_unmapped_resources( - project, matched_extensions=None, logger=None, to_queryset=None + project, matched_extensions=None, logger=None, to_queryset=None, **kwargs ): """ Match resources with empty status to PurlDB, if unmatched @@ -1636,7 +1646,7 @@ def flag_deployed_from_resources_with_missing_license(project, doc_extensions=No unknown_license_files.update(status=flag.UNKNOWN_LICENSE) -def handle_dangling_deployed_legal_files(project, logger, to_queryset=None): +def handle_dangling_deployed_legal_files(project, logger, to_queryset=None, **kwargs): """ Scan the legal files with empty status and update status to `REVIEW_DANGLING_LEGAL_FILE`. @@ -1892,7 +1902,7 @@ def is_invalid_match(match, matched_path_length): return matched_path_length == 1 and len(match.resource_ids) != 1 -def map_elfs_with_dwarf_paths(project, logger=None, to_queryset=None): +def map_elfs_with_dwarf_paths(project, logger=None, to_queryset=None, **kwargs): """Map ELF binaries to their sources in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -1964,7 +1974,7 @@ def get_go_file_paths(location): return file_paths -def map_go_paths(project, logger=None, to_queryset=None): +def map_go_paths(project, logger=None, to_queryset=None, **kwargs): """Map Go binaries to their source in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -2020,7 +2030,7 @@ def map_go_paths(project, logger=None, to_queryset=None): WINPE_BINARY_OPTIONS = ["Windows"] -def extract_binary_symbols(project, options, logger=None, to_queryset=None): +def extract_binary_symbols(project, options, logger=None, to_queryset=None, **kwargs): """ Extract binary symbols for all Elf, Mach0 and Winpe binaries found in the ``project`` resources, based on selected @@ -2065,7 +2075,7 @@ def extract_binary_symbols(project, options, logger=None, to_queryset=None): ) -def map_rust_binaries_with_symbols(project, logger=None, to_queryset=None): +def map_rust_binaries_with_symbols(project, logger=None, to_queryset=None, **kwargs): """Map Rust binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -2093,7 +2103,7 @@ def map_rust_binaries_with_symbols(project, logger=None, to_queryset=None): ) -def map_go_binaries_with_symbols(project, logger=None, to_queryset=None): +def map_go_binaries_with_symbols(project, logger=None, to_queryset=None, **kwargs): """Map Go binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -2121,7 +2131,7 @@ def map_go_binaries_with_symbols(project, logger=None, to_queryset=None): ) -def map_elfs_binaries_with_symbols(project, logger=None, to_queryset=None): +def map_elfs_binaries_with_symbols(project, logger=None, to_queryset=None, **kwargs): """Map Elf binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -2146,7 +2156,7 @@ def map_elfs_binaries_with_symbols(project, logger=None, to_queryset=None): ) -def map_macho_binaries_with_symbols(project, logger=None, to_queryset=None): +def map_macho_binaries_with_symbols(project, logger=None, to_queryset=None, **kwargs): """Map macho binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -2174,7 +2184,7 @@ def map_macho_binaries_with_symbols(project, logger=None, to_queryset=None): ) -def map_winpe_binaries_with_symbols(project, logger=None, to_queryset=None): +def map_winpe_binaries_with_symbols(project, logger=None, to_queryset=None, **kwargs): """Map winpe binaries to their source using symbols in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() if to_queryset is not None: @@ -2268,7 +2278,7 @@ def extract_binary_symbols_from_resources(resources, binary_symbols_func, logger logger(f"Error parsing binary symbols at: {resource.location_path!r} {e!r}") -def map_javascript_symbols(project, logger=None, to_queryset=None): +def map_javascript_symbols(project, logger=None, to_queryset=None, **kwargs): """Map deployed JavaScript, TypeScript to its sources using symbols.""" project_files = project.codebaseresources.files() @@ -2369,7 +2379,7 @@ def _map_javascript_symbols(to_resource, javascript_from_resources, logger): return 0 -def map_javascript_strings(project, logger=None, to_queryset=None): +def map_javascript_strings(project, logger=None, to_queryset=None, **kwargs): """Map deployed JavaScript, TypeScript to its sources using string literals.""" project_files = project.codebaseresources.files() @@ -2457,7 +2467,7 @@ def _map_javascript_strings(to_resource, javascript_from_resources, logger): return 0 -def map_python_pyx_to_binaries(project, logger=None, to_queryset=None): +def map_python_pyx_to_binaries(project, logger=None, to_queryset=None, **kwargs): """Map Cython source to their compiled binaries in ``project``.""" from source_inspector.symbols_tree_sitter import get_tree_and_language_info @@ -2505,7 +2515,7 @@ def map_python_pyx_to_binaries(project, logger=None, to_queryset=None): ) -def map_python_protobuf_files(project, logger=None, to_queryset=None): +def map_python_protobuf_files(project, logger=None, to_queryset=None, **kwargs): """Map protobuf-generated .py/.pyi files to their source .proto files.""" from_resources = ( project.codebaseresources.files().from_codebase().filter(extension=".proto")