Skip to content

Commit cfb9140

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Merge branch 'develop' into improve-license-detection
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>
2 parents 4f85513 + 742b20c commit cfb9140

File tree

100 files changed

+753
-51
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+753
-51
lines changed

‎CHANGELOG.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ v33.0.0 (next next, roadmap)
4646
- Update ABOUT files to adapt the ABOUT File Specification.
4747
See https://github.com/aboutcode-org/scancode-toolkit/issues/4181
4848

49+
- Also return sha1_git checksums for each files with ``--info`` plugin.
50+
https://github.com/aboutcode-org/scancode-toolkit/issues/624
51+
4952

5053
v32.3.3 - 2025-03-06
5154
--------------------

‎azure-pipelines.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ jobs:
203203
python_architecture: x64
204204
test_suites:
205205
click_versions: |
206-
for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.3 8.0.2 8.0.1 7.1.2 7.1.1 7.1 6.7;
206+
for clk_ver in 8.2.0 8.2.1 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7;
207207
do
208208
venv/bin/pip install click==$clk_ver;
209209
venv/bin/scancode -i samples/ -n3 --json foo.json;

‎docs/source/cli-reference/output-format.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ following options.
141141
"date": "2019-02-12",
142142
"sha1": "fca4540d490fff36bb90fd801cf9cd8fc695bb17",
143143
"md5": "a980b61c1e8be68d5cdb1236ba6b43e7",
144+
"sha1_git": "d9a10c0d8e868ebf8da0b3dc95bb0be634c34bfe",
144145
"mime_type": "text/x-c++",
145146
"file_type": "C++ source, ASCII text",
146147
"programming_language": "C++",
@@ -481,6 +482,7 @@ Comparing Different ``json`` Output Formats
481482
- size,
482483
- sha1,
483484
- md5,
485+
- sha1_git,
484486
- files_count,
485487
- mime_type,
486488
- file_type,

‎docs/source/cli-reference/scan-options-post.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1060,6 +1060,7 @@ To see all plugins available via command line help, use ``--plugins``.
10601060
"sha1": null,
10611061
"md5": null,
10621062
"sha256": null,
1063+
"sha1_git": null,
10631064
"mime_type": null,
10641065
"file_type": null,
10651066
"programming_language": null,

‎docs/source/cli-reference/synopsis.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ A sample JSON output file structure will look like::
156156
"date": "2019-02-12",
157157
"sha1": "2e07e32c52d607204fad196052d70e3d18fb8636",
158158
"md5": "effc6856ef85a9250fb1a470792b3f38",
159+
"sha1_git": "d9a10c0d8e868ebf8da0b3dc95bb0be634c34bfe",
159160
"mime_type": "text/plain",
160161
"file_type": "ASCII text",
161162
"programming_language": null,

‎requirements.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@ certifi==2024.2.2
88
cffi==1.16.0
99
chardet==5.0.0
1010
charset-normalizer==2.1.0
11-
click==8.1.7
11+
click==8.2.1;python_version>='3.10'
12+
click==8.1.7;python_version<'3.10'
1213
colorama==0.4.5
13-
commoncode==32.2.1
14+
commoncode==32.3.0
1415
construct==2.10.68
1516
container-inspector==31.1.0
1617
cryptography==42.0.5

‎setup-mini.cfg

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,10 @@ install_requires =
6767
Beautifulsoup4[chardet] >= 4.13.0
6868
boolean.py >= 4.0
6969
chardet >= 3.0.0
70-
click >= 6.7, !=7.0, !=8.1.8
70+
click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'
71+
click >= 8.2.0;python_version>='3.10'
7172
colorama >= 0.3.9
72-
commoncode >= 32.2.1
73+
commoncode >= 32.3.0
7374
container-inspector >= 31.0.0
7475
debian-inspector >= 31.1.0
7576
dparse2 >= 0.7.0

‎setup.cfg

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,10 @@ install_requires =
6767
Beautifulsoup4[chardet] >= 4.13.0
6868
boolean.py >= 4.0
6969
chardet >= 3.0.0
70-
click >= 6.7, !=7.0, !=8.1.8
70+
click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'
71+
click >= 8.2.0;python_version>='3.10'
7172
colorama >= 0.3.9
72-
commoncode >= 32.2.1
73+
commoncode >= 32.3.0
7374
container-inspector >= 31.0.0
7475
debian-inspector >= 31.1.0
7576
dparse2 >= 0.7.0

‎src/licensedcode/data/licenses/code-credit-license-1.0-0.LICENSE renamed to ‎src/licensedcode/data/licenses/code-credit-license-1.0.0.LICENSE

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
key: code-credit-license-1.0-0
2+
key: code-credit-license-1.0.0
33
short_name: Code Credit License v1.0.0
44
name: Code Credit License v1.0.0
55
category: Permissive
@@ -114,4 +114,4 @@ No contributor can revoke this license.
114114
***As far as the law allows, this software comes as is,
115115
without any warranty or condition, and no contributor
116116
will be liable to anyone for any damages related to this
117-
software or this license, under any kind of legal claim.***
117+
software or this license, under any kind of legal claim.***

‎src/licensedcode/data/licenses/liliq-r-1.1.LICENSE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
key: liliq-r-1.1
33
language: fr
44
short_name: LiLiQ-R-1.1
5-
name: Licence Libre du Québec – Réciprocité version 1.
5+
name: Licence Libre du Québec – Réciprocité version 1.1
66
category: Copyleft Limited
77
owner: Quebec
88
homepage_url: https://opensource.org/licenses/LiLiQ-R-1.1

‎src/licensedcode/detection.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,15 @@ class UniqueDetection:
905905
detection_log = attr.ib(default=attr.Factory(list))
906906
file_regions = attr.ib(factory=list)
907907

908+
@property
909+
def is_unknown(self):
910+
"""
911+
Return True if there are unknown license keys in the license expression
912+
for this detection, return False otherwise. By design these are licenses with "unknown" in
913+
their key.
914+
"""
915+
return 'unknown' in self.license_expression
916+
908917
@classmethod
909918
def get_unique_detections(cls, license_detections):
910919
"""

‎src/scancode/api.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,10 +330,14 @@ def get_file_info(location, **kwargs):
330330
result['date'] = get_last_modified_date(location) or None
331331
result['size'] = getsize(location) or 0
332332

333-
sha1, md5, sha256 = multi_checksums(location, ('sha1', 'md5', 'sha256')).values()
333+
sha1, md5, sha256, sha1_git = multi_checksums(
334+
location=location,
335+
checksum_names=('sha1', 'md5', 'sha256', 'sha1_git')
336+
).values()
334337
result['sha1'] = sha1
335338
result['md5'] = md5
336339
result['sha256'] = sha256
340+
result['sha1_git'] = sha1_git
337341

338342
collector = get_type(location)
339343
result['mime_type'] = collector.mimetype_file or None

‎src/scancode/plugin_info.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class InfoScanner(ScanPlugin):
2727
('sha1', attr.ib(default=None, repr=False)),
2828
('md5', attr.ib(default=None, repr=False)),
2929
('sha256', attr.ib(default=None, repr=False)),
30+
('sha1_git', attr.ib(default=None, repr=False)),
3031
('mime_type', attr.ib(default=None, repr=False)),
3132
('file_type', attr.ib(default=None, repr=False)),
3233
('programming_language', attr.ib(default=None, repr=False)),

‎tests/cluecode/data/plugin_filter_clues/filtered-expected.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"sha1": "c4d6f133413f1ea9d3c436fb797f9060c8640c04",
3535
"md5": "55a970437979ffe95e7b8ee6e51abb0c",
3636
"sha256": "ebdef41bf0f5bd7d378e4f76052e02156ce1c80ce67f89209b018a3fca8a071c",
37+
"sha1_git": "c6cc3f85376fd76bdebcc4fee6e9137e80bd8691",
3738
"mime_type": "text/plain",
3839
"file_type": "ASCII text",
3940
"programming_language": null,

‎tests/cluecode/data/plugin_filter_clues/filtered-expected2.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"sha1": "c43ffed01990c30300e07231790ace9f103ff7d7",
3535
"md5": "9946f7dcaf5066f2fb17e682e0326090",
3636
"sha256": "58416604a57e2e2625b802ac8c751c61763ae3d279c9c75d7c3bb55a3782cc6c",
37+
"sha1_git": "9956de3f204ca2d5c46b77e70c4c6bf9509f6dd8",
3738
"mime_type": "text/plain",
3839
"file_type": "ASCII text",
3940
"programming_language": null,

‎tests/cluecode/data/plugin_filter_clues/filtered-expected3.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"sha1": "9db841f62fdb2372f5f474bb75366140a00c62ca",
3535
"md5": "3fdbe85d79aa58aaeab119505f2007b6",
3636
"sha256": "408dfe8f9a70ea7bc9feaa9f77fb731ada0c26dc39008fdc5ead52d0442c656f",
37+
"sha1_git": "3ed0b925a62cbabad29cda7743c513d2ed557dc2",
3738
"mime_type": "text/plain",
3839
"file_type": "ASCII text",
3940
"programming_language": "verilog",

‎tests/cluecode/data/plugin_filter_clues/filtered-expected4.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@
146146
"sha1": "8a2e40b4eb23cc05a0b78330d919a7ffacde7a9a",
147147
"md5": "d5d03e14130735213e0532277df33cab",
148148
"sha256": "e4bd79e88b577d66d351597c0d4114b9ee2e31f0544795e87a88ca1b0dd38383",
149+
"sha1_git": "e7c0b65bcdb17888027d0b53a1f65ad61b238245",
149150
"mime_type": "text/plain",
150151
"file_type": "ASCII text",
151152
"programming_language": null,

‎tests/formattedcode/data/common/manifests-expected.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1018,6 +1018,7 @@
10181018
"sha1": null,
10191019
"md5": null,
10201020
"sha256": null,
1021+
"sha1_git": null,
10211022
"mime_type": null,
10221023
"file_type": null,
10231024
"programming_language": null,
@@ -1052,6 +1053,7 @@
10521053
"sha1": null,
10531054
"md5": null,
10541055
"sha256": null,
1056+
"sha1_git": null,
10551057
"mime_type": null,
10561058
"file_type": null,
10571059
"programming_language": null,
@@ -1088,6 +1090,7 @@
10881090
"sha1": "82107f6781df8da9aa7cb60abc649208256af5cb",
10891091
"md5": "e79938fe4e630b02e3526c0195292cc6",
10901092
"sha256": "f94648bd92ddf2464e62c56c3833b36927fa8b9803e8ebf3e4cfa7f2d797f3ad",
1093+
"sha1_git": "614191600be349e61331a3a0eb2b883493f0db0f",
10911094
"mime_type": "text/xml",
10921095
"file_type": "XML 1.0 document, ASCII text",
10931096
"programming_language": null,
@@ -1216,6 +1219,7 @@
12161219
"sha1": null,
12171220
"md5": null,
12181221
"sha256": null,
1222+
"sha1_git": null,
12191223
"mime_type": null,
12201224
"file_type": null,
12211225
"programming_language": null,
@@ -1250,6 +1254,7 @@
12501254
"sha1": "0ed61bf6912407a143ce96c95bea56ad13463f5b",
12511255
"md5": "427e6d3b579b128fed50328ac65b698e",
12521256
"sha256": "d292aa647dc82e007ba319c878ec182d6f235f0a923cfe012ffc0ae8b7fa23be",
1257+
"sha1_git": "fee016dc24204e079cb9e0217d4439d73e1398ac",
12531258
"mime_type": "application/json",
12541259
"file_type": "JSON data",
12551260
"programming_language": null,
@@ -1550,6 +1555,7 @@
15501555
"sha1": null,
15511556
"md5": null,
15521557
"sha256": null,
1558+
"sha1_git": null,
15531559
"mime_type": null,
15541560
"file_type": null,
15551561
"programming_language": null,
@@ -1584,6 +1590,7 @@
15841590
"sha1": "e6acddf907b7089d197d9d46879cd18ffb8f5198",
15851591
"md5": "5ba816691af915c02567f459ae26ebd6",
15861592
"sha256": "11129bc3bbf44b2567307ad13a31bc231f08416919e36bbdeabb9195b3e6ad58",
1593+
"sha1_git": "99128a9e08f5a23c650d83af5dcf40c3b976753d",
15871594
"mime_type": "application/json",
15881595
"file_type": "JSON data",
15891596
"programming_language": null,
@@ -1774,6 +1781,7 @@
17741781
"sha1": null,
17751782
"md5": null,
17761783
"sha256": null,
1784+
"sha1_git": null,
17771785
"mime_type": null,
17781786
"file_type": null,
17791787
"programming_language": null,
@@ -1808,6 +1816,7 @@
18081816
"sha1": "5507192502255517e006c531d07dd84dc2325be8",
18091817
"md5": "6795bed0c9c6b6cb915bfa959a2d1d4a",
18101818
"sha256": "4378d297df9fcb4f65fdcd1299e17eef7db86883baf149bc41b0335f943cf006",
1819+
"sha1_git": "887bd3bfe3bdba0c8f737a69df38454b3d489ac1",
18111820
"mime_type": "text/x-script.python",
18121821
"file_type": "Python script, ASCII text executable",
18131822
"programming_language": "Python",

‎tests/formattedcode/data/common/manifests-expected.jsonlines

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020
"system_environment": {
2121
"operating_system": "linux",
2222
"cpu_architecture": "64",
23-
"platform": "Linux-6.8.0-52-generic-x86_64-with-glibc2.35",
24-
"platform_version": "#53~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Jan 15 19:18:46 UTC 2",
25-
"python_version": "3.12.9 (main, Feb 20 2025, 19:13:20) [GCC 11.4.0]"
23+
"platform": "Linux-5.15.0-141-generic-x86_64-with-glibc2.35",
24+
"platform_version": "#151-Ubuntu SMP Sun May 18 21:35:19 UTC 2025",
25+
"python_version": "3.10.12 (main, Feb 4 2025, 14:57:36) [GCC 11.4.0]"
2626
},
2727
"spdx_license_list_version": "3.26",
2828
"files_count": 4
@@ -1056,6 +1056,7 @@
10561056
"sha1": null,
10571057
"md5": null,
10581058
"sha256": null,
1059+
"sha1_git": null,
10591060
"mime_type": null,
10601061
"file_type": null,
10611062
"programming_language": null,
@@ -1094,6 +1095,7 @@
10941095
"sha1": null,
10951096
"md5": null,
10961097
"sha256": null,
1098+
"sha1_git": null,
10971099
"mime_type": null,
10981100
"file_type": null,
10991101
"programming_language": null,
@@ -1134,6 +1136,7 @@
11341136
"sha1": "82107f6781df8da9aa7cb60abc649208256af5cb",
11351137
"md5": "e79938fe4e630b02e3526c0195292cc6",
11361138
"sha256": "f94648bd92ddf2464e62c56c3833b36927fa8b9803e8ebf3e4cfa7f2d797f3ad",
1139+
"sha1_git": "614191600be349e61331a3a0eb2b883493f0db0f",
11371140
"mime_type": "text/xml",
11381141
"file_type": "XML 1.0 document, ASCII text",
11391142
"programming_language": null,
@@ -1266,6 +1269,7 @@
12661269
"sha1": null,
12671270
"md5": null,
12681271
"sha256": null,
1272+
"sha1_git": null,
12691273
"mime_type": null,
12701274
"file_type": null,
12711275
"programming_language": null,
@@ -1304,6 +1308,7 @@
13041308
"sha1": "0ed61bf6912407a143ce96c95bea56ad13463f5b",
13051309
"md5": "427e6d3b579b128fed50328ac65b698e",
13061310
"sha256": "d292aa647dc82e007ba319c878ec182d6f235f0a923cfe012ffc0ae8b7fa23be",
1311+
"sha1_git": "fee016dc24204e079cb9e0217d4439d73e1398ac",
13071312
"mime_type": "application/json",
13081313
"file_type": "JSON data",
13091314
"programming_language": null,
@@ -1608,6 +1613,7 @@
16081613
"sha1": null,
16091614
"md5": null,
16101615
"sha256": null,
1616+
"sha1_git": null,
16111617
"mime_type": null,
16121618
"file_type": null,
16131619
"programming_language": null,
@@ -1646,6 +1652,7 @@
16461652
"sha1": "e6acddf907b7089d197d9d46879cd18ffb8f5198",
16471653
"md5": "5ba816691af915c02567f459ae26ebd6",
16481654
"sha256": "11129bc3bbf44b2567307ad13a31bc231f08416919e36bbdeabb9195b3e6ad58",
1655+
"sha1_git": "99128a9e08f5a23c650d83af5dcf40c3b976753d",
16491656
"mime_type": "application/json",
16501657
"file_type": "JSON data",
16511658
"programming_language": null,
@@ -1840,6 +1847,7 @@
18401847
"sha1": null,
18411848
"md5": null,
18421849
"sha256": null,
1850+
"sha1_git": null,
18431851
"mime_type": null,
18441852
"file_type": null,
18451853
"programming_language": null,
@@ -1878,6 +1886,7 @@
18781886
"sha1": "5507192502255517e006c531d07dd84dc2325be8",
18791887
"md5": "6795bed0c9c6b6cb915bfa959a2d1d4a",
18801888
"sha256": "4378d297df9fcb4f65fdcd1299e17eef7db86883baf149bc41b0335f943cf006",
1889+
"sha1_git": "887bd3bfe3bdba0c8f737a69df38454b3d489ac1",
18811890
"mime_type": "text/x-script.python",
18821891
"file_type": "Python script, ASCII text executable",
18831892
"programming_language": "Python",

0 commit comments

Comments
 (0)