Skip to content

Commit edcfb4f

Browse files
authored
Merge pull request #422 from CIRCL/misp_modules
Misp modules
2 parents d5f6ffe + 7c7f367 commit edcfb4f

File tree

21 files changed

+367
-119
lines changed

21 files changed

+367
-119
lines changed

OVERVIEW.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,24 @@ Redis and ARDB overview
9797
| ------ | ------ | ------ |
9898
| ail:all_role | **role** | **int, role priority (1=admin)** |
9999

100+
##### MISP Modules:
101+
102+
| Set Key | Value |
103+
| ------ | ------ |
104+
| enabled_misp_modules | **module name** |
105+
106+
| Key | Value |
107+
| ------ | ------ |
108+
| misp_module:**module name** | **module dict** |
109+
100110
##### Item Import:
101111
| Key | Value |
102112
| ------ | ------ |
103113
| **uuid**:isfile | **boolean** |
104114
| **uuid**:paste_content | **item_content** |
105115

116+
## DB2 - TermFreq:
117+
106118
| Set Key | Value |
107119
| ------ | ------ |
108120
| submitted:uuid | **uuid** |

bin/lib/MispModules.py

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#!/usr/bin/python3
2+
3+
import os
4+
import json
5+
import redis
6+
import requests
7+
import configparser
8+
9+
misp_module_url = 'http://localhost:6666'
10+
11+
default_config_path = os.path.join(os.environ['AIL_HOME'], 'configs', 'misp_modules.cfg')
12+
13+
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
14+
import ConfigLoader
15+
16+
config_loader = ConfigLoader.ConfigLoader()
17+
r_serv = config_loader.get_redis_conn("ARDB_DB")
18+
config_loader = None
19+
20+
def init_config(config_path=default_config_path):
21+
config = configparser.ConfigParser()
22+
if os.path.isfile(config_path):
23+
config.read(config_path)
24+
else:
25+
config.add_section('misp_modules')
26+
config.set('misp_modules', 'url', 'http://localhost')
27+
config.set('misp_modules', 'port', '6666')
28+
return config
29+
30+
def init_module_config(module_json, config, config_path=default_config_path):
31+
if 'config' in module_json['meta']:
32+
if module_json['meta']['config']:
33+
if module_json['name'] not in config:
34+
config.add_section(module_json['name'])
35+
for config_var in module_json['meta']['config']:
36+
if config_var not in config[module_json['name']]:
37+
config.set(module_json['name'], config_var, '')
38+
return config
39+
40+
def load_modules_list():
41+
req = requests.get('{}/modules'.format(misp_module_url))
42+
if req.status_code == 200:
43+
all_misp_modules = req.json()
44+
all_modules = []
45+
for module_json in all_misp_modules:
46+
47+
#filter module-types
48+
if 'hover' in module_json['meta']['module-type'] or 'expansion' in module_json['meta']['module-type']:
49+
all_modules.append(module_json)
50+
51+
# # TODO: handle import/export modules
52+
53+
config = init_config()
54+
r_serv.delete('misp_modules')
55+
for module_json in all_modules:
56+
config = init_module_config(module_json, config, config_path=default_config_path)
57+
r_serv.hset('misp_modules', module_json['name'], json.dumps(module_json))
58+
59+
with open(default_config_path, 'w') as f:
60+
config.write(f)
61+
62+
else:
63+
print('Error: Module service not reachable.')
64+
65+
66+
def build_config_json(module_name):
67+
misp_module_config = configparser.ConfigParser()
68+
misp_module_config.read(default_config_path)
69+
dict_config = {}
70+
if module_name in misp_module_config:
71+
for config_key in misp_module_config[module_name]:
72+
config_value = misp_module_config[module_name][config_key]
73+
if config_value:
74+
dict_config[config_key] = config_value
75+
return dict_config
76+
77+
def build_enrichment_request_json(module_name, var_name, var_value):
78+
# # TODO: add error handler
79+
request_dict = {'module': module_name, var_name: var_value}
80+
# add config
81+
config_json = build_config_json(module_name)
82+
if config_json:
83+
request_dict['config'] = config_json
84+
return json.dumps(request_dict)
85+
86+
def misp_module_enrichment_request(misp_module_url, misp_module_port, request_content):
87+
# # TODO: check if module is enabled
88+
endpoint_url = '{}:{}/query'.format(misp_module_url, misp_module_port)
89+
req = requests.post(endpoint_url, headers={'Content-Type': 'application/json'}, data=request_content)
90+
if req.status_code == 200:
91+
response = req.json()
92+
if response:
93+
return parse_module_enrichment_response(response)
94+
else:
95+
print('error: {} Enrichment service not reachable.'.format(req.status_code,))
96+
return ''
97+
98+
def parse_module_enrichment_response(misp_module_response):
99+
print(misp_module_response)
100+
response_values = []
101+
if 'results' in misp_module_response:
102+
# # TODO: handle misp_format (Attribute, Object, Tags)
103+
response_types = []
104+
for result in misp_module_response['results']:
105+
# get all types
106+
for resp_type in result['types']:
107+
response_types.append(resp_type)
108+
# get all values
109+
for resp_value in result['values']:
110+
response_values.append(resp_value)
111+
# TODO: handle / verify / use response types
112+
#print(response_types)
113+
return response_values
114+
115+
if __name__ == "__main__":
116+
117+
load_modules_list()
118+
119+
misp_module_url = 'http://localhost'
120+
misp_module_port = 6666
121+
122+
bitcoin_address = 'bitcoin_address'
123+
test_content = build_enrichment_request_json('btc_steroids', 'btc', bitcoin_address)
124+
print(test_content)
125+
misp_module_enrichment_request(misp_module_url, misp_module_port, test_content)

update/v2.5/Update.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env python3
2+
# -*-coding:UTF-8 -*
3+
4+
import os
5+
import re
6+
import sys
7+
import time
8+
import redis
9+
import datetime
10+
11+
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
12+
import ConfigLoader
13+
14+
new_version = 'v2.5'
15+
16+
if __name__ == '__main__':
17+
18+
start_deb = time.time()
19+
20+
config_loader = ConfigLoader.ConfigLoader()
21+
r_serv = config_loader.get_redis_conn("ARDB_DB")
22+
config_loader = None
23+
24+
r_serv.zadd('ail:all_role', 3, 'user')
25+
r_serv.zadd('ail:all_role', 4, 'user_no_api')
26+
r_serv.zadd('ail:all_role', 5, 'read_only')
27+
28+
for user in r_serv.hkeys(user:all):
29+
r_serv.sadd('user_role:user', user)
30+
r_serv.sadd('user_role:user_no_api', user)
31+
r_serv.sadd('user_role:read_only', user)
32+
33+
#Set current ail version
34+
r_serv.set('ail:version', new_version)
35+
36+
#Set current ail version
37+
r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v2.5/Update.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
3+
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
4+
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
5+
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
6+
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
7+
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
8+
9+
export PATH=$AIL_HOME:$PATH
10+
export PATH=$AIL_REDIS:$PATH
11+
export PATH=$AIL_ARDB:$PATH
12+
export PATH=$AIL_BIN:$PATH
13+
export PATH=$AIL_FLASK:$PATH
14+
15+
GREEN="\\033[1;32m"
16+
DEFAULT="\\033[0;39m"
17+
18+
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
19+
bash ${AIL_BIN}/LAUNCH.sh -ks
20+
wait
21+
22+
bash ${AIL_BIN}/LAUNCH.sh -lav &
23+
wait
24+
echo ""
25+
26+
echo ""
27+
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
28+
echo ""
29+
python ${AIL_HOME}/update/v2.5/Update.py
30+
wait
31+
echo ""
32+
echo ""
33+
34+
echo ""
35+
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
36+
bash ${AIL_BIN}/LAUNCH.sh -ks
37+
wait
38+
39+
exit 0

var/www/blueprints/correlation.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
# Import Role_Manager
2020
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
21-
from Role_Manager import login_admin, login_analyst
21+
from Role_Manager import login_admin, login_analyst, login_read_only
2222

2323
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
2424
import Correlate_object
@@ -121,7 +121,7 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal
121121
# ============= ROUTES ==============
122122
@correlation.route('/correlation/show_correlation', methods=['GET', 'POST']) # GET + POST
123123
@login_required
124-
@login_analyst
124+
@login_read_only
125125
def show_correlation():
126126
if request.method == 'POST':
127127
object_type = request.form.get('object_type')
@@ -197,7 +197,7 @@ def show_correlation():
197197

198198
@correlation.route('/correlation/graph_node_json')
199199
@login_required
200-
@login_analyst
200+
@login_read_only
201201
def graph_node_json(): # # TODO: use post
202202
correlation_id = request.args.get('correlation_id')
203203
type_id = request.args.get('type_id')

var/www/blueprints/crawler_splash.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
# Import Role_Manager
2020
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
21-
from Role_Manager import login_admin, login_analyst
21+
from Role_Manager import login_admin, login_analyst, login_read_only
2222

2323
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
2424
import Tag
@@ -47,7 +47,7 @@ def api_validator(api_response):
4747
# add route : /crawlers/show_domain
4848
@crawler_splash.route('/crawlers/showDomain')
4949
@login_required
50-
@login_analyst
50+
@login_read_only
5151
def showDomain():
5252
domain_name = request.args.get('domain')
5353
epoch = request.args.get('epoch')

var/www/create_default_user.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@
2323
if not r_serv.exists('ail:all_role'):
2424
r_serv.zadd('ail:all_role', 1, 'admin')
2525
r_serv.zadd('ail:all_role', 2, 'analyst')
26+
r_serv.zadd('ail:all_role', 3, 'user')
27+
r_serv.zadd('ail:all_role', 4, 'user_no_api')
28+
r_serv.zadd('ail:all_role', 5, 'read_only')
2629

2730
username = '[email protected]'
2831
password = gen_password()

var/www/modules/Role_Manager.py

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from flask import request, make_response, current_app
1717

1818
login_manager = LoginManager()
19-
login_manager.login_view = 'role'
19+
login_manager.login_view = 'root.role'
2020

2121
# CONFIG #
2222
config_loader = ConfigLoader.ConfigLoader()
@@ -68,7 +68,35 @@ def decorated_view(*args, **kwargs):
6868
return func(*args, **kwargs)
6969
return decorated_view
7070

71+
def login_user(func):
72+
@wraps(func)
73+
def decorated_view(*args, **kwargs):
74+
if not current_user.is_authenticated:
75+
return login_manager.unauthorized()
76+
elif (not current_user.is_in_role('user')):
77+
return login_manager.unauthorized()
78+
return func(*args, **kwargs)
79+
return decorated_view
7180

81+
def login_user_no_api(func):
82+
@wraps(func)
83+
def decorated_view(*args, **kwargs):
84+
if not current_user.is_authenticated:
85+
return login_manager.unauthorized()
86+
elif (not current_user.is_in_role('user_no_api')):
87+
return login_manager.unauthorized()
88+
return func(*args, **kwargs)
89+
return decorated_view
90+
91+
def login_read_only(func):
92+
@wraps(func)
93+
def decorated_view(*args, **kwargs):
94+
if not current_user.is_authenticated:
95+
return login_manager.unauthorized()
96+
elif (not current_user.is_in_role('read_only')):
97+
return login_manager.unauthorized()
98+
return func(*args, **kwargs)
99+
return decorated_view
72100

73101
###############################################################
74102
###############################################################
@@ -107,11 +135,15 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
107135
# create user token
108136
generate_new_token(username_id)
109137

138+
if not role:
139+
role = 'read_only'
140+
110141
if update:
111142
r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
112143
# remove default user password file
113144
if username_id=='[email protected]':
114145
os.remove(default_passwd_file)
146+
r_serv_db.hset('user:all', username_id, password_hash)
115147
else:
116148
if default:
117149
r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
@@ -121,7 +153,7 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
121153
r_serv_db.sadd('user_role:{}'.format(role_to_add), username_id)
122154
r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role)
123155

124-
r_serv_db.hset('user:all', username_id, password_hash)
156+
r_serv_db.hset('user:all', username_id, password_hash)
125157

126158
def edit_user_db(user_id, role, password=None):
127159
if password:

0 commit comments

Comments
 (0)