Merge branch 'main' of https://github.com/ambuj-1211/vulnerablecode into add-cwe-support-in-multiple-importers

Author: ambuj-1211

.github/workflows/pypi-release.yml

@@ -37,7 +37,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist/
 
       - name: Upload built archives
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pypi_archives
           path: dist/*
@@ -51,7 +51,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist

CHANGELOG.rst

@@ -4,11 +4,23 @@ Release notes
 Version (next)
 -------------------
 
+
+Version v34.0.2
+-------------------
+
+- Add management command to commit exported vulnerability data (#1600)
+- Fix API 500 error (#1603)
+
+
+Version v34.0.1
+-------------------
+
 - Add Pipeline to flag ghost packages (#1533)
 - Add logging configuration (#1533)
 - Drop support for python 3.8 (#1533)
 - Drop using docker-compose and use the built-in "docker compose" instead
 - Upgrade core dependencies including Django and Rest Framework
+- Fix typo in KEV improver (#1594)
 
 
 Version v34.0.0

aboutcode/hashid/__init__.py

@@ -28,7 +28,7 @@
 which makes every filesystem performance suffer.
 
 In addition, when storing these files in Git repositories, we need to avoid creating any repository
-with too many files that would make using this repository impactical or exceed the limits of some
+with too many files that would make using this repository impractical or exceed the limits of some
 repository hosting services.
 
 Therefore we are storing vulnerability data using a directory tree using the first few characters
@@ -46,21 +46,21 @@ def build_vcid(prefix="VCID"):
     """
     Return a new Vulnerable Code ID (aka. VCID) which is a strongly unique vulnerability
     identifier string using the provided ``prefix``. A VCID is composed of a four letter prefix, and
-    three segments composed of four letters and dihits each separated by a dash.
+    three segments composed of four letters and digits each separated by a dash.
     For example::
     >>> import re
     >>> vcid = build_vcid()
     >>> assert re.match('VCID(-[a-hjkm-z1-9]{4}){3}', vcid), vcid
 
     We were mistakenly not using enough bits. The symptom was that the last
-    segment of the VCID was always strting with "aaa" This ensure we are now OK:
+    segment of the VCID was always string with "aaa" This ensure we are now OK:
     >>> vcids = [build_vcid() for _ in range(50)]
     >>> assert not any(vid.split("-")[-1].startswith("aaa") for vid in vcids)
     """
     uid = uuid4().bytes
-    # we keep  three segments of 4 base32-encodee bytes, 3*4=12
+    # we keep  three segments of 4 base32-encoded bytes, 3*4=12
     # which corresponds to 60 bits
-    # becausee each base32 byte can store 5 bits (2**5 = 32)
+    # because each base32 byte can store 5 bits (2**5 = 32)
     uid = base32_custom(uid)[:12].decode("utf-8").lower()
     return f"{prefix}-{uid[:4]}-{uid[4:8]}-{uid[8:12]}"
 
@@ -72,7 +72,7 @@ def get_vcid_yml_file_path(vcid: str):
     return Path(VULNERABILITY_REPO_NAME) / vulnerability_yml_path(vcid)
 
 
-# This cuxstom 32 characters alphabet is designed to avoid visually easily confusable characters:
+# This custom 32 characters alphabet is designed to avoid visually easily confusable characters:
 # i and l
 # 0 and o
 _base32_alphabet = b"abcdefghjkmnpqrstuvwxyz123456789"
@@ -117,7 +117,7 @@ def vulnerability_yml_path(vcid):
     Return the path to a vulnerability YAML file crafted from the ``vcid`` VCID vulnerability id.
 
     The approach is to distribute the files in many directories to avoid having too many files in
-    any directory and be able to find the path to a vulneravility file given its VCID distributed on
+    any directory and be able to find the path to a vulnerability file given its VCID distributed on
     the first two characters of the UUID section of a VCID.
 
     The UUID is using a base32 encoding, hence keeping two characters means 32 x 32 = 1024
@@ -140,9 +140,12 @@ def get_package_base_dir(purl: Union[PackageURL, str]):
     """
     Return the base path to a Package directory (ignoring version) for a purl
     """
+    if isinstance(purl, str):
+        purl = PackageURL.from_string(purl)
+
     path_elements = package_path_elements(purl)
     phash, core_path, _pversion, _extra_path = path_elements
-    return Path(f"{PACKAGE_REPOS_NAME_PREFIX}-{phash}") / core_path
+    return Path(f"{PACKAGE_REPOS_NAME_PREFIX}-{purl.type}-{phash}") / core_path
 
 
 def get_package_purls_yml_file_path(purl: Union[PackageURL, str]):
@@ -159,6 +162,52 @@ def get_package_vulnerabilities_yml_file_path(purl: Union[PackageURL, str]):
     return get_package_base_dir(purl) / VULNERABILITIES_FILENAME
 
 
+# We use a 4-tier system for storing package metadata.
+# The tiers are as follows:
+# 1. Super Large Ecosystem (~5M packages): 2^10 = 1,024 git repositories
+# 2. Large Ecosystem (~500K packages): 2^7 = 128 git repositories
+# 3. Medium Ecosystem (~50K packages): 2^5 = 32 git repositories
+# 4. Small Ecosystem (~2K packages): 2^0 = 1 git repository
+# See https://github.com/aboutcode-org/federatedcode/issues/3#issuecomment-2388371726
+BIT_COUNT_BY_ECOSYSTEM = {
+    # Super Large Ecosystem
+    "github": 10,
+    "npm": 10,
+    # Large Ecosystem
+    "golang": 7,
+    "maven": 7,
+    "nuget": 7,
+    "perl": 7,
+    "php": 7,
+    "pypi": 7,
+    "ruby": 7,
+    # Medium Ecosystem
+    "alpm": 5,
+    "bitbucket": 5,
+    "cocoapods": 5,
+    "composer": 5,
+    "deb": 5,
+    "docker": 5,
+    "gem": 5,
+    "generic": 5,
+    "huggingface": 5,
+    "mlflow": 5,
+    "pub": 5,
+    "rpm": 5,
+    # Small Ecosystem
+    "bitnami": 0,
+    "cargo": 0,
+    "conan": 0,
+    "conda": 0,
+    "cpan": 0,
+    "cran": 0,
+    "hackage": 0,
+    "hex": 0,
+    "luarocks": 0,
+    "swift": 0,
+}
+
+
 def package_path_elements(purl: Union[PackageURL, str]):
     """
     Return 4-tuple of POSIX path strings crafted from the ``purl`` package PURL string or object.
@@ -196,7 +245,7 @@ def package_path_elements(purl: Union[PackageURL, str]):
           sbom.spdx.2.2.json : a SPDX SBOM
           .... other files
 
-          <extra_path> : one sub directory for each quote-encoded <qualifiers#supath> if any
+          <extra_path> : one sub directory for each quote-encoded <qualifiers#subpath> if any
             metadata.yml : ABOUT YAML file with package origin and license metadata for this version
             scancode-scan.yml : a scancode scan for this package version
             foo-scan.yml : a scan for this package version created with tool foo
@@ -208,15 +257,15 @@ def package_path_elements(purl: Union[PackageURL, str]):
     We keep the same prefix for different versions::
 
     >>> package_path_elements("pkg:pypi/[email protected]")
-    ('1050', 'pypi/license-expression', '30.3.1', '')
+    ('50', 'pypi/license-expression', '30.3.1', '')
     >>> package_path_elements("pkg:pypi/[email protected]")
-    ('1050', 'pypi/license-expression', '10.3.1', '')
+    ('50', 'pypi/license-expression', '10.3.1', '')
 
     We encode with quotes, avoid double encoding of already quoted parts to make subpaths easier
     for filesystems::
 
     >>> package_path_elements("pkg:pypi/[email protected]?foo=bar&baz=bar#sub/path")
-    ('1050', 'pypi/license-expression', '30.3.1', 'baz%3Dbar%26foo%3Dbar%23sub%2Fpath')
+    ('50', 'pypi/license-expression', '30.3.1', 'baz%3Dbar%26foo%3Dbar%23sub%2Fpath')
 
     >>> purl = PackageURL(
     ...     type="pypi",
@@ -225,12 +274,13 @@ def package_path_elements(purl: Union[PackageURL, str]):
     ...     qualifiers=dict(foo="bar"),
     ...     subpath="a/b/c")
     >>> package_path_elements(purl)
-    ('1050', 'pypi/license-expression', 'b%23ar%2F%3F30.3.2%21', 'foo%3Dbar%23a%2Fb%2Fc')
+    ('50', 'pypi/license-expression', 'b%23ar%2F%3F30.3.2%21', 'foo%3Dbar%23a%2Fb%2Fc')
     """
     if isinstance(purl, str):
         purl = PackageURL.from_string(purl)
 
-    purl_hash = get_purl_hash(purl)
+    bit_count = BIT_COUNT_BY_ECOSYSTEM.get(purl.type, 0)
+    purl_hash = get_purl_hash(purl=purl, _bit_count=bit_count)
 
     if ns := purl.namespace:
         ns_name = f"{ns}/{purl.name}"
@@ -287,17 +337,17 @@ def get_core_purl(purl: Union[PackageURL, str]):
     return PackageURL(**purld)
 
 
-def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 13) -> str:
+def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 0) -> str:
     """
     Return a short lower cased hash string from a ``purl`` string or object. The PURL is normalized
     and we drop its version, qualifiers and subpath.
 
-    This function takes a normalized PURL string and a ``_bit_count`` argument defaulting to 13 bits
-    which represents 2**13 = 8192 possible hash values. It returns a fixed length short hash string
+    This function takes a normalized PURL string and a ``_bit_count`` argument defaulting to 0 bits
+    which represents 2**0 = 1 possible hash value. It returns a fixed length short hash string
     that is left-padded with zeros.
 
     The hash length is derived from the bit_count and the number of bits-per-byte stored in an hex
-    encoding of this bits count. For 13 bits, this means up to 4 characters.
+    encoding of this bits count. For 10 bits, this means up to 3 characters.
 
     The function is carefully designed to be portable across tech stacks and easy to implement in
     many programming languages:
@@ -319,23 +369,23 @@ def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 13) -> str:
     For example::
 
     The hash does not change with version or qualifiers::
-    >>> get_purl_hash("pkg:pypi/[email protected]")
-    '1289'
-    >>> get_purl_hash("pkg:pypi/[email protected]")
-    '1289'
-    >>> get_purl_hash("pkg:pypi/[email protected]?foo=bar#sub/path")
-    '1289'
+    >>> get_purl_hash("pkg:pypi/[email protected]", 7)
+    '09'
+    >>> get_purl_hash("pkg:pypi/[email protected]", 7)
+    '09'
+    >>> get_purl_hash("pkg:pypi/[email protected]?foo=bar#sub/path", 7)
+    '09'
 
     The hash is left padded with zero if it::
-    >>> get_purl_hash("pkg:pypi/expressionss")
-    '0057'
+    >>> get_purl_hash("pkg:pypi/expressionss", 7)
+    '57'
 
     We normalize the PURL. Here pypi normalization always uses dash for underscore ::
 
-    >>> get_purl_hash("pkg:pypi/license_expression")
-    '1050'
-    >>> get_purl_hash("pkg:pypi/license-expression")
-    '1050'
+    >>> get_purl_hash("pkg:pypi/license_expression", 7)
+    '50'
+    >>> get_purl_hash("pkg:pypi/license-expression", 7)
+    '50'
 
     Originally from:
     https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154

aboutcode/hashid/tests/test_hashid.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# Portions Copyright (c) The Python Software Foundation
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0 and Python-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import pytest
+
+from aboutcode.hashid import package_path_elements
+
+
+@pytest.mark.parametrize(
+    "purl, purl_hash",
+    [
+        ("pkg:maven/org.apache.commons/io", "4f"),
+        ("pkg:GOLANG/google.golang.org/genproto@abcdedf#/googleapis/api/annotations/", "4a"),
+        ("pkg:golang/github.com/nats-io/nats-server/v2/[email protected]", "22"),
+        ("pkg:bitbucket/birKenfeld/pyGments-main@244fd47e07d1014f0aed9c", "03"),
+        ("pkg:github/Package-url/purl-Spec@244fd47e07d1004f0aed9c", "095"),
+        ("pkg:deb/debian/[email protected]?arch=i386&distro=jessie", "19"),
+        (
+            "pkg:docker/customer/dockerimage@sha256:244fd47e07d1004f0aed9c?repository_url=gcr.io",
+            "10",
+        ),
+        ("pkg:gem/[email protected]?Platform=java", "1e"),
+        (
+            "pkg:Maven/org.apache.xmlgraphics/[email protected]?repositorY_url=repo.spring.io/release&classifier=sources",
+            "28",
+        ),
+        (
+            "pkg:Maven/org.apache.xmlgraphics/[email protected]?repositorY_url=repo.spring.io/release&extension=pom",
+            "28",
+        ),
+        ("pkg:Maven/net.sf.jacob-project/[email protected]?type=dll&classifier=x86", "17"),
+        ("pkg:npm/%40angular/[email protected]", "323"),
+        ("pkg:Nuget/[email protected]", "63"),
+        ("pkg:PYPI/[email protected]", "00"),
+        ("pkg:composer/guzzlehttp/[email protected]", "1d"),
+        ("pkg:Rpm/fedora/[email protected]?Arch=i386&Distro=fedora-25", "16"),
+        ("pkg:maven/HTTPClient/[email protected]", "4d"),
+        ("pkg:maven/mygroup/[email protected]%20Final?mykey=my%20value", "6f"),
+        ("pkg:npm/@babel/core#/googleapis/api/annotations/", "0dc"),
+        ("pkg:npm/@babel/[email protected]#/googleapis/api/annotations/", "0dc"),
+        ("pkg:npm/[email protected]#/googleapis/api/annotations/", "23b"),
+        ("pkg:npm/core#/googleapis/api/annotations/", "23b"),
+    ],
+)
+def test_purl_hash(purl, purl_hash):
+    result_hash, *_ = package_path_elements(purl)
+    assert result_hash == purl_hash

pyproject-aboutcode.hashid.toml

@@ -66,6 +66,7 @@ excludes = [
     "**/*.bak",
     "**/.ipynb_checkpoints",
     "aboutcode/hashid/python.LICENSE",
+    "aboutcode/hashid/tests/**/*",
 ]
 
 metadata_files = ["apache-2.0.LICENSE", "NOTICE",  "aboutcode/hashid/python.LICENSE"]

requirements.txt

@@ -21,12 +21,13 @@ click==8.1.2
 coreapi==2.3.3
 coreschema==0.0.4
 cryptography==43.0.1
+crispy-bootstrap4==2024.1
 cwe2==3.0.0
 dateparser==1.1.1
 decorator==5.1.1
 defusedxml==0.7.1
 distro==1.7.0
-Django==4.2.15
+Django==4.2.16
 django-crispy-forms==2.3
 django-environ==0.11.2
 django-filter==24.3
@@ -35,10 +36,10 @@ djangorestframework==3.15.2
 doc8==0.11.1
 docopt==0.6.2
 docutils==0.17.1
-drf-spectacular==0.27.2
-drf-spectacular-sidecar==2024.7.1 
+drf-spectacular==0.24.2
+drf-spectacular-sidecar==2022.10.1
 executing==0.8.3
-fetchcode==0.3.0
+fetchcode==0.6.0
 freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 34.0.0
+version = 34.0.2
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -62,11 +62,12 @@ install_requires =
     django-filter>=24.0
     django-widget-tweaks>=1.5.0
     django-crispy-forms>=2.3
+    crispy-bootstrap4>=2024.1
     django-environ>=0.11.0
     gunicorn>=23.0.0
 
     # for the API doc
-    drf-spectacular[sidecar]>=0.27.2
+    drf-spectacular[sidecar]>=0.24.2
 
     #essentials
     packageurl-python>=0.15
@@ -89,7 +90,7 @@ install_requires =
     # networking
     GitPython>=3.1.17
     requests>=2.25.1
-    fetchcode>=0.3.0
+    fetchcode>=0.6.0
 
     #pipeline
     aboutcode.pipeline>=0.1.0