Skip to content

Commit 9ce80aa

Browse files
AndrewMohawkAndrewMohawk
committed
Added python file to generate JSON from CVE dataset and datatables html file to visualise it
1 parent ee35bde commit 9ce80aa

File tree

3 files changed

+258
-0
lines changed

3 files changed

+258
-0
lines changed

‎summary_html/CVE_list.json

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎summary_html/cve_summary.html

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
<!DOCTYPE html>
2+
<html>
3+
4+
<head>
5+
<meta content="initial-scale=1, maximum-scale=1,
6+
user-scalable=0" name="viewport" />
7+
<meta name="viewport" content="width=device-width" />
8+
9+
<!-- Stylesheets for bootstrap + datatables 5 -->
10+
<link rel='stylesheet' href='https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.0.1/css/bootstrap.min.css'>
11+
<link rel='stylesheet' href='https://cdn.datatables.net/1.11.4/css/dataTables.bootstrap5.min.css'>
12+
<link rel="stylesheet" href="https://cdn.datatables.net/1.11.4/js/dataTables.bootstrap5.min.js">
13+
14+
<!-- JS for jquery + datatables + datatables>bootstrap-->
15+
16+
<script type="text/javascript" src="https://code.jquery.com/jquery-3.5.1.js">
17+
</script>
18+
<script type="text/javascript" src="https://cdn.datatables.net/1.11.4/js/jquery.dataTables.min.js">
19+
</script>
20+
<script type="text/javascript" src="https://cdn.datatables.net/1.11.4/js/dataTables.bootstrap5.min.js">
21+
</script>
22+
23+
24+
</script>
25+
26+
<style>
27+
body {
28+
margin-top: 5%;
29+
font-size:12px;
30+
margin-left:5%;
31+
margin-right:5%;
32+
}
33+
34+
#overlay {
35+
background: #ffffff;
36+
color: #666666;
37+
position: fixed;
38+
height: 100%;
39+
width: 100%;
40+
z-index: 5000;
41+
top: 0;
42+
left: 0;
43+
float: left;
44+
text-align: center;
45+
padding-top: 25%;
46+
opacity: .80;
47+
}
48+
49+
button {
50+
margin: 40px;
51+
padding: 5px 20px;
52+
cursor: pointer;
53+
}
54+
55+
.spinner {
56+
margin: 0 auto;
57+
height: 64px;
58+
width: 64px;
59+
animation: rotate 0.8s infinite linear;
60+
border: 5px solid firebrick;
61+
border-right-color: transparent;
62+
border-radius: 50%;
63+
}
64+
65+
@keyframes rotate {
66+
0% {
67+
transform: rotate(0deg);
68+
}
69+
70+
100% {
71+
transform: rotate(360deg);
72+
}
73+
}
74+
</style>
75+
</head>
76+
77+
<body>
78+
79+
<!-- Bootstrap overlay div with growing spinner -->
80+
<div id="overlay">
81+
<div class="spinner"></div>
82+
<br />
83+
Loading... (usually 5-10s)
84+
</div>
85+
86+
87+
<div class="row justify-content-center" width="800px">
88+
<div class="">
89+
<h2>CVE Search</h2>
90+
<!--HTML table with student data-->
91+
<table id="datatable" class="table table-striped table-bordered" style="display:none" cellspacing="0"
92+
width="100%">
93+
<thead>
94+
<tr>
95+
<th>CVE_Year</th>
96+
<th>CVE_Name</th>
97+
<th>CVE_description</th>
98+
<th>CVE_github</th>
99+
<th>CVE_references</th>
100+
</tr>
101+
</thead>
102+
<tfoot>
103+
<tr>
104+
<th>CVE_Year</th>
105+
<th>CVE_Name</th>
106+
<th>CVE_description</th>
107+
<th>CVE_github</th>
108+
<th>CVE_references</th>
109+
</tr>
110+
</tfoot>
111+
112+
</table>
113+
</div>
114+
</div>
115+
116+
<footer class="mt-auto text-black-50">
117+
<p>Searchable CVEs from <a href="https://github.com/trickest/cve" class="text">https://github.com/trickest/cve</a>, by <a href="https://twitter.com/andrewmohawk" class="text">@AndrewMohawk</a>.</p>
118+
</footer>
119+
120+
<script type="text/javascript" src="CVE_list.json"></script>
121+
122+
123+
<script>
124+
125+
/* Initialization of datatable */
126+
$(document).ready(function () {
127+
table_settings = {
128+
"dom": 'flirtp<"dt-buttons"Bf>',
129+
"paging": true,
130+
"processing": true,
131+
responsive: true,
132+
oLanguage: { sProcessing: "<div id='loader'></div>" },
133+
"buttons": [
134+
'colvis',
135+
],
136+
"bSortClasses": false,
137+
"bAutoWidth": false,
138+
"aoColumns": [
139+
{ "sWidth": "5%" },
140+
{ "sWidth": "10%" },
141+
{ "sWidth": "55%" },
142+
{ "sWidth": "20%" },
143+
{ "sWidth": "10%" },
144+
],
145+
"order": [[ 1, "desc" ]],
146+
"data": dataTable_data,
147+
initComplete: function () {
148+
// Apply the search
149+
this.api().columns().every(function () {
150+
var that = this;
151+
152+
$('input', this.footer()).on('keyup change clear', function () {
153+
if (that.search() !== this.value) {
154+
that
155+
.search(this.value)
156+
.draw();
157+
}
158+
});
159+
});
160+
161+
162+
}
163+
}
164+
165+
$('#datatable tfoot th').each(function () {
166+
var title = $(this).text();
167+
$(this).html('<input type="text" placeholder="Search ' + title + '" />');
168+
});
169+
170+
$('#overlay').show()
171+
$('#datatable').on('init.dt', function () {
172+
173+
console.log('Table initialisation complete: ' + new Date().getTime());
174+
$('#datatable').show();
175+
$('#overlay').hide()
176+
})
177+
.on('init', function () {
178+
$('*[type="search"][class="form-control input-sm"]')
179+
.addClass('input-lg')
180+
.css({ 'width': '400px', 'display': 'inline-block' });
181+
$('div.dataTables_filter').css({ 'margin-top': '1em' });
182+
})
183+
.dataTable(table_settings);
184+
185+
});
186+
</script>
187+
</body>
188+
189+
</html>

‎summary_html/generate_summary_json.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
2+
#!/usr/bin/python3
3+
import os
4+
import datetime
5+
import sys
6+
import json
7+
import re
8+
9+
#Path to enumerate CVEs from
10+
dir = "../"
11+
CVE_list = []
12+
13+
14+
#fetch all the years
15+
years = os.listdir(dir)
16+
#remove non numeric years
17+
years = [year for year in years if year.isdigit()]
18+
#sort descending (we want the latest at the top)
19+
years.sort(reverse=True)
20+
21+
#clean up the text blocks
22+
def clean_text(description):
23+
24+
#remove the '-' at the beginning of each line
25+
description_lines = description.split('\n')
26+
description_lines = [line.lstrip('- ') for line in description_lines]
27+
28+
#change urls with <a> links with regular expression
29+
description_lines = [re.sub(r'(https?:\/\/[^\s]+)', r'<a target="_blank" href="\1">\1</a>', line) for line in description_lines]
30+
31+
#add <br/> for each line
32+
description = '<br/>'.join(description_lines)
33+
return description
34+
35+
36+
37+
#generate JSON for each CVE
38+
for year in years:
39+
40+
yearDir = os.path.join(dir, year)
41+
for CVE_filename in os.listdir(yearDir):
42+
43+
#open CVE file
44+
CVE_file = open(os.path.join(yearDir, CVE_filename), 'r')
45+
#read CVE file
46+
CVE_file_content = CVE_file.read()
47+
48+
#extract CVE description, references and github
49+
CVE_description = CVE_file_content.split('### Description')[1].split('###')[0].strip()
50+
CVE_references = CVE_file_content.split('### Reference')[1].split('###')[0].strip()
51+
CVE_github = CVE_file_content.split('### Github')[1].split('###')[0].strip()
52+
53+
#TODO: extract imageshield label attributes
54+
55+
CVE_Name = CVE_filename.split('.')[0]
56+
57+
CVE_description = clean_text(CVE_description)
58+
CVE_github = clean_text(CVE_github)
59+
CVE_references = clean_text(CVE_references)
60+
61+
thisCVE = [year,CVE_Name, CVE_description, CVE_github,CVE_references]
62+
CVE_list.append(thisCVE)
63+
64+
CVE_output = f"dataTable_data = {json.dumps(CVE_list)}"
65+
66+
#save CVE list to JSON file
67+
with open('CVE_list.json', 'w') as outfile:
68+
outfile.write(CVE_output)

0 commit comments

Comments
 (0)