CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' into crawler_manager

pull/559/head
Terrtia 2021-03-05 18:02:08 +01:00
parent 1f94c1c693 5ee1303db4
commit 2b082d01a0
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
6 changed files with 45 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bin/Helper.py
View File

@ -150,8 +150,11 @@ class Process(object):
def populate_set_in(self):
# monoproc
src = self.modules.get(self.subscriber_name, 'subscribe')
if src != 'Redis':
try:
src = self.modules.get(self.subscriber_name, 'subscribe')
except configparser.NoOptionError: #NoSectionError
src = None
if src != 'Redis' and src:
self.pubsub.setup_subscribe(src)
for msg in self.pubsub.subscribe():
in_set = self.subscriber_name + 'in'
@ -159,7 +162,7 @@ class Process(object):
self.r_temp.hset('queues', self.subscriber_name,
int(self.r_temp.scard(in_set)))
else:
print('{} has no suscriber'.format(self.subscriber_name))
print('{} has no subscriber'.format(self.subscriber_name))
def get_from_set(self):
# multiproc

7
bin/IPAddress.py
View File

@ -62,9 +62,14 @@ if __name__ == '__main__':
p = Process(config_section)
ip_networks = []
networks = p.config.get("IP", "networks")
if not networks:
print('No IP ranges provided')
sys.exit(0)
try:
for network in p.config.get("IP", "networks").split(","):
for network in networks.split(","):
ip_networks.append(IPv4Network(network))
print(f'IP Range: {network}')
except:
print('Please provide a list of valid IP addresses')
sys.exit(0)

2
bin/TermTrackerMod.py
View File

@ -81,7 +81,7 @@ if __name__ == "__main__":
try:
dict_words_freq = Term.get_text_word_frequency(item_content)
except TimeoutException:
print ("{0} processing timeout".format(paste.p_rel_path))
print ("{0} processing timeout".format(item_id))
continue
else:
signal.alarm(0)

15
bin/lib/Domain.py
View File

@ -246,10 +246,11 @@ def sanithyse_domain_name_to_search(name_to_search, domain_type):
if domain_type == 'onion':
r_name = r'[a-z0-9\.]+'
else:
r_name = r'[a-zA-Z0-9\.-_]+'
r_name = r'[a-zA-Z0-9-_\.]+'
# invalid domain name
if not re.fullmatch(r_name, name_to_search):
return None
res = re.match(r_name, name_to_search)
return {'search': name_to_search, 'error': res.string.replace( res[0], '')}
return name_to_search.replace('.', '\.')
@ -257,7 +258,7 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
domains_dict = {}
for domain_type in domain_types:
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
if not name_to_search:
if not name_to_search or isinstance(r_name, dict):
break
r_name = re.compile(r_name)
for domain in get_all_domains_up(domain_type):
@ -269,6 +270,14 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
domains_dict[domain]['hl-end'] = res.end()
return domains_dict
def api_sanithyse_domain_name_to_search(name_to_search, domains_types):
domains_types = sanitize_domain_types(domains_types)
for domain_type in domains_types:
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
if isinstance(r_name, dict):
return ({'error': 'Invalid'}, 400)
def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1):
domains_types = sanitize_domain_types(domains_types)
domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)

1
bin/packages/modules.cfg
View File

@ -3,7 +3,6 @@ subscribe = ZMQ_Global
publish = Redis_Mixer,Redis_preProcess1
[Importer_Json]
subscribe = ZMQ_JSON
publish = Redis_Mixer,Redis_Tags
[Global]

36
var/www/modules/restApi/Flask_restApi.py
View File

@ -68,6 +68,10 @@ def get_user_from_token(token):
return r_serv_db.hget('user:tokens', token)
def verify_user_role(role, token):
# User without API
if role == 'user_no_api':
return False
user_id = get_user_from_token(token)
if user_id:
if is_in_role(user_id, role):
@ -188,14 +192,14 @@ def one():
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_item_id():
data = request.get_json()
res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/default", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_item_id_basic():
data = request.get_json()
@ -218,7 +222,7 @@ def get_item_id_basic():
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item/tag", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_item_tag():
data = request.get_json()
@ -299,7 +303,7 @@ def delete_item_tags():
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item/content", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_item_content():
data = request.get_json()
@ -314,7 +318,7 @@ def get_item_content():
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/tag/metadata", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_tag_metadata():
data = request.get_json()
tag = data.get('tag', None)
@ -324,7 +328,7 @@ def get_tag_metadata():
return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
@restApi.route("api/v1/get/tag/all", methods=['GET'])
@token_required('user')
@token_required('read_only')
def get_all_tags():
res = {'tags': Tag.get_all_tags()}
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
@ -351,7 +355,7 @@ def delete_tracker_term():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/tracker/item", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_tracker_term_item():
data = request.get_json()
user_token = get_auth_from_header()
@ -364,7 +368,7 @@ def get_tracker_term_item():
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_cryptocurrency_bitcoin_metadata():
data = request.get_json()
crypto_address = data.get('bitcoin', None)
@ -373,7 +377,7 @@ def get_cryptocurrency_bitcoin_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_cryptocurrency_bitcoin_item():
data = request.get_json()
bitcoin_address = data.get('bitcoin', None)
@ -385,7 +389,7 @@ def get_cryptocurrency_bitcoin_item():
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_pgp_key_metadata():
data = request.get_json()
pgp_field = data.get('key', None)
@ -394,7 +398,7 @@ def get_pgp_key_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_pgp_mail_metadata():
data = request.get_json()
pgp_field = data.get('mail', None)
@ -403,7 +407,7 @@ def get_pgp_mail_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_pgp_name_metadata():
data = request.get_json()
pgp_field = data.get('name', None)
@ -412,7 +416,7 @@ def get_pgp_name_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_pgp_key_item():
data = request.get_json()
pgp_field = data.get('key', None)
@ -421,7 +425,7 @@ def get_pgp_key_item():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_pgp_mail_item():
data = request.get_json()
pgp_mail = data.get('mail', None)
@ -430,7 +434,7 @@ def get_pgp_mail_item():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
@token_required('user')
@token_required('read_only')
def get_pgp_name_item():
data = request.get_json()
pgp_name = data.get('name', None)
@ -609,7 +613,7 @@ def import_json_item():
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/ping", methods=['GET'])
@token_required('user')
@token_required('read_only')
def v1_ping():
return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200