Add new features, paths, partial downloads.

* Add possibility to specify path for chrome
* Add possibility to specify path for input and output fill
* Add possibility for filler to only download new data

Author: danjo133

fetcher.py

@@ -11,13 +11,34 @@
 
 import time
 import csv
+import argparse
 from datetime import datetime
 from selenium.webdriver import Chrome, ChromeOptions
 from selenium.webdriver.common.by import By
 
 hacktivity_url = 'https://hackerone.com/hacktivity/overview'
 page_loading_timeout = 10
 
+def create_argument_parser():
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        '--browser-binary',
+        type=str,
+        help='Path to browser binary (Chrome or Chromium)',
+        default='/Applications/Google Chrome.app/Contents/MacOS/Google Chrome')
+    argparser.add_argument(
+        '--input-data-file',
+        type=str,
+        help='Path to input data file',
+        default='data.csv'
+    )
+    argparser.add_argument(
+        '--output-data-file',
+        type=str,
+        help='Path to output data file',
+        default='data.csv'
+    )
+    return argparser
 
 def extract_reports(raw_reports):
     reports = []
@@ -46,15 +67,15 @@ def extract_reports(raw_reports):
     return reports
 
 
-def fetch():
+def fetch(commandline_args):
     options = ChromeOptions()
-    options.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+    options.binary_location = commandline_args.browser_binary
     options.add_argument('no-sandbox')
     options.add_argument('headless')
     driver = Chrome(options=options)
 
     reports = []
-    with open('data.csv', 'r', newline='', encoding='utf-8') as file:
+    with open(commandline_args.input_data_file, 'r', newline='', encoding='utf-8') as file:
         reader = csv.DictReader(file)
         for row in reader:
             reports.append(dict(row))
@@ -93,12 +114,14 @@ def fetch():
     finally:
         driver.close()
 
-    with open('data.csv', 'w', newline='', encoding='utf-8') as file:
+    with open(commandline_args.output_data_file, 'w', newline='', encoding='utf-8') as file:
         keys = reports[0].keys()
         writer = csv.DictWriter(file, fieldnames=keys)
         writer.writeheader()
         writer.writerows(reports)
 
 
 if __name__ == '__main__':
-    fetch()
+    parser = create_argument_parser()
+    args = parser.parse_args()
+    fetch(args)

filler.py

@@ -11,37 +11,67 @@
 import csv
 import requests
 import time
+import argparse
 
-def fill():
-    reports = []
-    with open('data.csv', 'r', newline='', encoding='utf-8') as file:
+def create_argument_parser():
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        '--update-all',
+        action='store_true',
+        help='Update all reports',
+        default=False
+    )
+    argparser.add_argument(
+        '--input-data-file',
+        type=str,
+        help='Path to input data file',
+        default='data.csv'
+    )
+    argparser.add_argument(
+        '--output-data-file',
+        type=str,
+        help='Path to output data file',
+        default='data.csv'
+    )
+    return argparser
+
+def fill(commandline_args):
+    fetched_reports = []
+    new_reports = []
+    with open(commandline_args.input_data_file, 'r', newline='', encoding='utf-8') as file:
         reader = csv.DictReader(file)
         for row in reader:
-            reports.append(dict(row))
-    count_of_reports = len(reports)
+            if row['title'] == '' or commandline_args.update_all:
+                new_reports.append(dict(row))
+            else:
+                fetched_reports.append(dict(row))
+    count_of_reports = len(new_reports)
     for i in range(count_of_reports):
         time.sleep(0.5)
         print('Fetching report ' + str(i + 1) + ' out of ' + str(count_of_reports))
-        report_url = 'https://' + reports[i]['link'] + '.json'
+        report_url = 'https://' + new_reports[i]['link'] + '.json'
         try:
             json_info = requests.get(report_url).json()
-            reports[i]['title'] = json_info['title']
-            reports[i]['program'] = json_info['team']['profile']['name']
-            reports[i]['upvotes'] = int(json_info['vote_count'])
-            reports[i]['bounty'] = float(json_info['bounty_amount'] if 'bounty_amount' in json_info else "0") if json_info['has_bounty?'] else 0.0
-            reports[i]['vuln_type'] = json_info['weakness']['name'] if 'weakness' in json_info else ''
+            new_reports[i]['title'] = json_info['title']
+            new_reports[i]['program'] = json_info['team']['profile']['name']
+            new_reports[i]['upvotes'] = int(json_info['vote_count'])
+            new_reports[i]['bounty'] = float(json_info['bounty_amount'] if 'bounty_amount' in json_info else "0") if json_info['has_bounty?'] else 0.0
+            new_reports[i]['vuln_type'] = json_info['weakness']['name'] if 'weakness' in json_info else ''
         except Exception as err:
             print('error at report ' + str(i + 1), err)
             continue
 
-        print(reports[i])
+        print(new_reports[i])
 
-    with open('data.csv', 'w', newline='', encoding='utf-8') as file:
+    with open(commandline_args.output_data_file, 'w', newline='', encoding='utf-8') as file:
+        reports = new_reports + fetched_reports
         keys = reports[0].keys()
         writer = csv.DictWriter(file, fieldnames=keys)
         writer.writeheader()
         writer.writerows(reports)
 
 
 if __name__ == '__main__':
-    fill()
+    parser = create_argument_parser()
+    args = parser.parse_args()
+    fill(args)

requirements.txt

@@ -1,2 +1,3 @@
 selenium
-requests
+requests
+argparse