update

Author: bbogen

setup.py

@@ -18,9 +18,9 @@ def finalize_options(self):
         install.finalize_options(self)
         if self.location is None:
             self.location = os.path.expanduser("~/vul4j_data")
-        # if os.path.exists(self.location):
-        #     print(f"ERROR: Directory already exists: {self.location}")
-        #     exit(1)
+        if os.path.exists(self.location):
+            print(f"ERROR: Directory already exists: {self.location}")
+            exit(1)
         os.environ["VUL4J_DATA"] = self.location
 
     def run(self):

vul4j/config.py

@@ -3,30 +3,37 @@
 from os.path import normpath, expanduser
 
 
-def get_config(section: str, config_name: str, default=""):
+def get_config(section: str, config_name: str, default: str = "") -> str:
+    """
+    Returns the value of the given config entry.
+    First the vul4j.ini file is checked,
+    if the value or the config file is not found,
+    the environment variables are checked for the provided name,
+    and if none of the two previous checks have a value, the default value is used.
+
+    :param section: section name in the config file
+    :param config_name: config entry name in the config file and/or in the env vars
+    :param default: default value if no config value is found
+    :return:    string value of the config entry
+    """
     config = configparser.ConfigParser()
-    config_path = os.path.join(os.environ.get("VUL4J_DATA", expanduser("~/vul4j_data")), "vul4j.ini")
-    config.read(config_path)
+    config_path = os.path.join(VUL4J_DATA, "vul4j.ini")
+    config.read(config_path, encoding="utf-8")
 
-    try:
-        value = config.get(section, config_name, fallback=None)
-        if value is None or value == "":
-            value = os.environ.get(config_name)
-
-        return value if value else default
-    except configparser.NoSectionError:
-        print(f"Vul4J config not found at {config_path}")
-        exit(1)
+    value = config.get(section, config_name, fallback=None)
+    if value is None or value == "":
+        value = os.environ.get(config_name)
+    return value if value else default
 
 
 # VUl4J
 VUL4J_DATA = normpath(os.environ.get("VUL4J_DATA", expanduser("~/vul4j_data")))
 VUL4J_GIT = normpath(get_config("VUL4J", "VUL4J_GIT"))
-VUL4J_COMMITS_URL = get_config("VUL4J", "VUL4J_COMMITS_URL")
 DATASET_PATH = normpath(get_config("VUL4J", "DATASET_PATH",
                                    os.path.join(VUL4J_GIT, "dataset", "vul4j_dataset.csv")))
+VUL4J_COMMITS_URL = get_config("VUL4J", "VUL4J_COMMITS_URL")
 LOG_TO_FILE = get_config("VUL4J", "LOG_TO_FILE", "1") == "1"
-FILE_LOG_LEVEL = get_config("VUL4J", "LOG_LEVEL", "DEBUG").upper()
+FILE_LOG_LEVEL = get_config("VUL4J", "FILE_LOG_LEVEL", "INFO").upper()
 
 # DIRS
 VUL4J_OUTPUT = normpath(get_config("DIRS", "VUL4J_WORKDIR", "VUL4J"))

vul4j/main.py

@@ -1,8 +1,8 @@
 import argparse
-from datetime import datetime
 import os.path
 import subprocess
 import sys
+from datetime import datetime
 
 from loguru import logger
 
@@ -15,76 +15,50 @@
 # logger
 def setup_logger(command: str, display_level: str = "INFO", file_level: str = "DEBUG"):
     log_filename = f"{datetime.now().strftime('%y%m%d_%H%M%S')}_{command}_{file_level}.log"
-
     logger.remove()
+    # STDOUT
     logger.add(sys.stdout,
                colorize=True,
                format="<cyan>{time:YYYY-MM-DD HH:mm:ss}</cyan> | <level>{message}</level>",
                diagnose=False,
                backtrace=False,
                level=display_level.upper())
+    # FILE
     logger.add(os.path.join(VUL4J_DATA, "logs", log_filename),
                format="<cyan>{time:YYYY-MM-DD HH:mm:ss}</cyan> | <level>{level}</level> | <level>{message}</level>",
                rotation="00:00",
                level=file_level.upper())
 
 
 @utils.log_frame("STATUS")
-def vul4j_status(args):
+def vul4j_status(_):
     utils.check_status()
 
 
 @utils.log_frame("CHECKOUT")
 def vul4j_checkout(args):
-    vul_id = args.id
-    output_dir = args.outdir
-
-    try:
-        vul4j.checkout(vul_id, output_dir)
-        return
-    except (vul4j.VulnerabilityNotFoundError, AssertionError) as err:
-        logger.error(err)
-    exit(1)
+    vul4j.checkout(args.id, args.outdir)
 
 
 @utils.log_frame("COMPILE")
 def vul4j_compile(args):
-    output_dir = args.outdir
-
     try:
-        vul4j.build(output_dir)
-        return
+        vul4j.build(args.outdir)
     except subprocess.CalledProcessError:
-        logger.error("Compile failed!")
-    except (vul4j.VulnerabilityNotFoundError, AssertionError) as err:
-        logger.error(err)
-    exit(1)
+        raise subprocess.CalledProcessError("Compile failed!")
 
 
 @utils.log_frame("TEST")
 def vul4j_test(args):
-    output_dir = args.outdir
-    batch_type = args.batchtype
-
     try:
-        vul4j.test(output_dir, batch_type)
+        vul4j.test(args.outdir, args.batchtype)
     except subprocess.CalledProcessError:
-        logger.error("Testing failed!")
-    except (vul4j.VulnerabilityNotFoundError, AssertionError) as err:
-        logger.error(err)
+        raise subprocess.CalledProcessError("Testing failed!")
 
 
 @utils.log_frame("APPLY")
 def vul4j_apply(args):
-    output_dir = args.outdir
-    version = args.version
-
-    try:
-        vul4j.apply(output_dir, version)
-    except AssertionError as err:
-        logger.error(err)
-    except vul4j.VulnerabilityNotFoundError as err:
-        logger.error(err)
+    vul4j.apply(args.outdir, args.version)
 
 
 @utils.log_frame("SAST")
@@ -120,18 +94,12 @@ def vul4j_sast(args):
 
 @utils.log_frame("REPRODUCE")
 def vul4j_reproduce(args):
-    vul_id = args.id
-    vul4j.reproduce(vul_id)
+    vul4j.reproduce(args.id)
 
 
 @utils.log_frame("INFO")
 def vul4j_info(args):
-    vul_id = args.id
-
-    try:
-        vul4j.get_info(vul_id)
-    except vul4j.VulnerabilityNotFoundError as err:
-        logger.error(err)
+    vul4j.get_info(args.id)
 
 
 @utils.log_frame("CLASSPATH")
@@ -156,7 +124,7 @@ def main(args=None):
 
     # STATUS
     status_parser = sub_parsers.add_parser("status",
-                                           help="Lists vul4j requirements and availability.")
+                                           help="Lists vul4j requirements and their availability.")
     status_parser.set_defaults(func=vul4j_status)
 
     # CHECKOUT
@@ -186,16 +154,16 @@ def main(args=None):
 
     # APPLY
     apply_parser = sub_parsers.add_parser('apply',
-                                          help="Apply the specified file version.")
+                                          help="Apply the specified file versions.")
     apply_parser.add_argument("-d", "--outdir", type=str,
                               help="The directory to which the vulnerability was checked out.", required=True)
     apply_parser.add_argument("-v", "--version", type=str,
-                              help="Version to apply", required=True)
+                              help="Version to apply.", required=True)
     apply_parser.set_defaults(func=vul4j_apply)
 
     # SAST
     sast_parser = sub_parsers.add_parser('sast',
-                                         help="Run spotbugs analysis.")
+                                         help="Run Spotbugs analysis.")
     sast_parser.add_argument("-d", "--outdir", type=str,
                              help="The directory to which the vulnerability was checked out.", required=True)
     sast_parser.add_argument("-v", "--versions", nargs='+',
@@ -227,7 +195,7 @@ def main(args=None):
 
     # GET SPOTBUGS
     spotbugs_parser = sub_parsers.add_parser("get-spotbugs",
-                                             help="Downloads Spotbugs into the user directory.")
+                                             help="Download Spotbugs into the user directory.")
     spotbugs_parser.add_argument("-l", "--location", type=str,
                                  help="Custom spotbugs installation path.", required=False)
     spotbugs_parser.set_defaults(func=get_spotbugs)

vul4j/spotbugs.py

@@ -46,7 +46,7 @@ def extract_attributes(cls, root):
         return attributes
 
 
-def run_spotbugs(output_dir: str, version=None, force_compile=False) -> list:
+def run_spotbugs(project_dir: str, version=None, force_compile=False) -> list:
     """
     Runs Spotbugs check on the project found in the provided directory.
     The project must contain a 'vulnerability_info.json' file.
@@ -56,40 +56,40 @@ def run_spotbugs(output_dir: str, version=None, force_compile=False) -> list:
     One can manually force recompilation by setting force_compile to True.
 
     The project's target folder is searched for artifacts.
-    The jar that ends in 'SNAPSHOT.jar' will be used for the Spotbugs analysis.
 
-    The method getter extracts the modified method names and their classes into the modifications.json file.
+    The modification extractor extracts the modified classes, class attributes and method names
+    into the modifications.json file.
     Then Spotbugs analysis is run.
 
-    The spotbugs_report.xml file is checked for warnings in the methods extracted by the method getter.
+    The spotbugs_report.xml file is checked for warnings in the modified code parts.
     The results are saved in the warnings.json file or warnings_version.json if a version was provided.
 
-    :param output_dir:  path to the projects directory
+    :param project_dir:  path to the projects directory
     :param version: version name, used for naming output files
     :param force_compile:   recompile project
     """
 
-    vul = vul4j.Vulnerability.from_json(output_dir)
+    vul = vul4j.Vulnerability.from_json(project_dir)
 
     assert vul.build_system == "Maven", f"Incompatible build system: {vul.build_system}"
 
     # create spotbugs directory
-    reports_dir = os.path.join(output_dir, VUL4J_OUTPUT, "spotbugs")
+    reports_dir = os.path.join(project_dir, VUL4J_OUTPUT, "spotbugs")
     os.makedirs(reports_dir, exist_ok=True)
     assert os.path.exists(reports_dir), "Failed to create spotbugs directory!"
     logger.debug("Spotbugs directory created!")
 
     # get module path where compiled jars are located
     failing_module = vul.failing_module
     if failing_module == "root":
-        module_path = output_dir
+        module_path = project_dir
     else:
-        module_path = os.path.join(output_dir, failing_module)
+        module_path = os.path.join(project_dir, failing_module)
     logger.debug(f"Module path: {module_path}")
 
     # find modified methods and their classes
     method_getter_output = os.path.join(reports_dir, "modifications.json")
-    method_getter_command = f"java -jar {MODIFICATION_EXTRACTOR_PATH} {output_dir} {method_getter_output}"
+    method_getter_command = f"java -jar {MODIFICATION_EXTRACTOR_PATH} {project_dir} {method_getter_output}"
     method_getter_log_path = os.path.join(reports_dir, "modifications.log")
     log_to_file = open(method_getter_log_path, "w", encoding="utf-8") if LOG_TO_FILE else subprocess.DEVNULL
     logger.debug(method_getter_command)
@@ -106,7 +106,7 @@ def run_spotbugs(output_dir: str, version=None, force_compile=False) -> list:
     # check for artifacts, compiling if necessary
     if force_compile:
         logger.debug("Forced compile")
-        vul4j.build(output_dir, version, clean=True)
+        vul4j.build(project_dir, version, clean=True)
 
     # select the correct jar from artifacts
     jar_path = get_artifact(module_path)

vul4j/utils.py

@@ -32,13 +32,9 @@ def wrapper(*args, **kwargs):
                 func(*args, **kwargs)
             except Exception as err:
                 logger.error(err)
+                exit(1)
             finally:
-                if os.path.exists(VUL4J_GIT):
-                    repo = git.Repo(VUL4J_GIT)
-                    repo.git.reset("--hard")
-                    repo.git.checkout("--")
-                    repo.git.clean("-fdx")
-                    repo.git.checkout("-f", "main")
+                reset_vul4j_git()
                 end = f" END {title} "
                 logger.info(end.center(60, "="))
 
@@ -47,6 +43,15 @@ def wrapper(*args, **kwargs):
     return decorator_log_frame
 
 
+def reset_vul4j_git():
+    if os.path.exists(VUL4J_GIT):
+        repo = git.Repo(VUL4J_GIT)
+        repo.git.reset("--hard")
+        repo.git.checkout("--")
+        repo.git.clean("-fdx")
+        repo.git.checkout("-f", "main")
+
+
 def check_status():
     """
     Checks availability of vul4j dependencies.
@@ -63,11 +68,12 @@ def check_status():
 
     # check java versions
     env = os.environ.copy()
+    java_version_command = "java -version"
 
     java7 = False
     if JAVA7_HOME:
         env["PATH"] = os.path.join(JAVA7_HOME, "bin") + os.pathsep + env["PATH"]
-        java7 = "1.7" in str(subprocess.run("java -version",
+        java7 = "1.7" in str(subprocess.run(java_version_command,
                                             shell=True,
                                             stdout=subprocess.PIPE,
                                             stderr=subprocess.STDOUT,
@@ -76,7 +82,7 @@ def check_status():
     java8 = False
     if JAVA8_HOME:
         env["PATH"] = os.path.join(JAVA8_HOME, "bin") + os.pathsep + env["PATH"]
-        java8 = "1.8" in str(subprocess.run("java -version",
+        java8 = "1.8" in str(subprocess.run(java_version_command,
                                             shell=True,
                                             stdout=subprocess.PIPE,
                                             stderr=subprocess.STDOUT,
@@ -85,14 +91,23 @@ def check_status():
     java11 = False
     if JAVA11_HOME:
         env["PATH"] = os.path.join(JAVA11_HOME, "bin") + os.pathsep + env["PATH"]
-        java11 = "11" in str(subprocess.run("java -version",
+        java11 = "11" in str(subprocess.run(java_version_command,
+                                            shell=True,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.STDOUT,
+                                            env=env))
+
+    java16 = False
+    if JAVA16_HOME:
+        env["PATH"] = os.path.join(JAVA16_HOME, "bin") + os.pathsep + env["PATH"]
+        java11 = "16" in str(subprocess.run(java_version_command,
                                             shell=True,
                                             stdout=subprocess.PIPE,
                                             stderr=subprocess.STDOUT,
                                             env=env))
 
     # check maven
-    maven = subprocess.run("mvn --version",
+    maven = subprocess.run("mvn -version",
                            shell=True,
                            stdout=subprocess.DEVNULL,
                            stderr=subprocess.DEVNULL).returncode == 0
@@ -120,6 +135,7 @@ def log_result(message: str, success: bool):
     log_result("Java 7", java7)
     log_result("Java 8", java8)
     log_result("Java 11", java11)
+    log_result("Java 16", java16)
     log_result("Maven", maven)
     log_result("Spotbugs", spotbugs)
     log_result("Spotbugs method getter", method_getter)
@@ -181,7 +197,7 @@ def clean_build(project_dir: str, build_system: str, env: dict) -> None:
 
 def get_java_home_env(java_version: str) -> dict:
     """
-    Returns JAVA_HOME location depending on the specified java version.
+    Returns a copy of *os.environ* where the specified java version and all other java options are set.
 
     :param java_version: java version
     :return: env with all java parameters set

vul4j/vul4j.ini

@@ -1,9 +1,9 @@
 [VUL4J]
-VUL4J_GIT = /vul4j-github
+VUL4J_GIT = /vul4j
+DATASET_PATH =
 VUL4J_COMMITS_URL = https://github.com/tuhh-softsec/vul4j/commits/
-DATASET_PATH = /vul4j-github/dataset/vul4j_dataset.csv
 LOG_TO_FILE = 1
-LOG_LEVEL = INFO
+FILE_LOG_LEVEL = INFO
 
 [DIRS]
 VUL4J_WORKDIR = VUL4J