CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [tests] fix tests

pull/594/head
Terrtia 2023-05-23 13:38:33 +02:00
parent c37a68d41d
commit 94d7eaf11d
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
6 changed files with 166 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,6 +3,7 @@
*.pyc *.pyc
*.swo *.swo
.idea .idea
.coverage
# Install Dirs # Install Dirs
AILENV AILENV

4
bin/LAUNCH.sh
View File

@ -573,11 +573,11 @@ function update_thirdparty {
function launch_tests() { function launch_tests() {
tests_dir=${AIL_HOME}/tests tests_dir=${AIL_HOME}/tests
bin_dir=${AIL_BIN} bin_dir=${AIL_BIN}
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase --exclude=test-zmq.py
} }
function reset_password() { function reset_password() {
echo -e "\t* Reseting UI admin password..." echo -e "\t* Resetting UI admin password..."
if checking_kvrocks && checking_redis; then if checking_kvrocks && checking_redis; then
python ${AIL_HOME}/var/www/create_default_user.py & python ${AIL_HOME}/var/www/create_default_user.py &
wait wait

8
bin/modules/ApiKey.py
View File

@ -51,10 +51,10 @@ class ApiKey(AbstractModule):
item = Item(item_id) item = Item(item_id)
item_content = item.get_content() item_content = item.get_content()
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content) google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True)
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content) aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True)
if aws_access_key: if aws_access_key:
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content) aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True)
if aws_access_key or google_api_key: if aws_access_key or google_api_key:
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};' to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
@ -74,7 +74,7 @@ class ApiKey(AbstractModule):
print(f'found AWS secret key') print(f'found AWS secret key')
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}') self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}')
msg = 'infoleak:automatic-detection="aws-key";{}'.format(item.get_id()) msg = f'infoleak:automatic-detection="aws-key";{item.get_id()}'
self.add_message_to_queue(msg, 'Tags') self.add_message_to_queue(msg, 'Tags')
# Tags # Tags

308
tests/testApi.py
View File

@ -14,159 +14,163 @@ from lib import Tag
from packages import Import_helper from packages import Import_helper
sys.path.append(os.environ['AIL_FLASK']) sys.path.append(os.environ['AIL_FLASK'])
from var.www.Flask_server import app sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
from Flask_server import app
def parse_response(obj, ail_response):
res_json = ail_response.get_json()
if 'status' in res_json:
if res_json['status'] == 'error':
return obj.fail('{}: {}: {}'.format(ail_response.status_code, res_json['status'], res_json['reason']))
return res_json
def get_api_key():
api_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
if os.path.isfile(api_file):
with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
content = f.read()
content = content.splitlines()
apikey = content[-1]
apikey = apikey.replace('API_Key=', '', 1)
# manual tests
else:
apikey = sys.argv[1]
return apikey
APIKEY = get_api_key() # def parse_response(obj, ail_response):
# res_json = ail_response.get_json()
class TestApiV1(unittest.TestCase): # if 'status' in res_json:
import_uuid = None # if res_json['status'] == 'error':
item_id = None # return obj.fail('{}: {}: {}'.format(ail_response.status_code, res_json['status'], res_json['reason']))
# return res_json
#
def setUp(self): #
self.app = app # def get_api_key():
self.app.config['TESTING'] = True # api_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
self.client = self.app.test_client() # if os.path.isfile(api_file):
self.apikey = APIKEY # with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
self.item_content = "text to import" # content = f.read()
self.item_tags = ["infoleak:analyst-detection=\"private-key\""] # content = content.splitlines()
self.expected_tags = ["infoleak:analyst-detection=\"private-key\"", 'infoleak:submission="manual"'] # apikey = content[-1]
# apikey = apikey.replace('API_Key=', '', 1)
# POST /api/v1/import/item # # manual tests
def test_0001_api_import_item(self): # else:
input_json = {"type": "text","tags": self.item_tags,"text": self.item_content} # apikey = sys.argv[1]
req = self.client.post('/api/v1/import/item', json=input_json ,headers={ 'Authorization': self.apikey }) # return apikey
req_json = parse_response(self, req) #
import_uuid = req_json['uuid'] #
self.__class__.import_uuid = import_uuid # APIKEY = get_api_key()
self.assertTrue(Import_helper.is_valid_uuid_v4(import_uuid)) #
#
# POST /api/v1/get/import/item # class TestApiV1(unittest.TestCase):
def test_0002_api_get_import_item(self): # import_uuid = None
input_json = {"uuid": self.__class__.import_uuid} # item_id = None
item_not_imported = True #
import_timout = 60 # def setUp(self):
start = time.time() # self.app = app
# self.app.config['TESTING'] = True
while item_not_imported: # self.client = self.app.test_client()
req = self.client.post('/api/v1/get/import/item', json=input_json ,headers={ 'Authorization': self.apikey }) # self.apikey = APIKEY
req_json = parse_response(self, req) # self.item_content = "text to import"
if req_json['status'] == 'imported': # self.item_tags = ["infoleak:analyst-detection=\"private-key\""]
try: # self.expected_tags = ["infoleak:analyst-detection=\"private-key\"", 'infoleak:submission="manual"']
item_id = req_json['items'][0] #
item_not_imported = False # # POST /api/v1/import/item
except Exception as e: # def test_0001_api_import_item(self):
if time.time() - start > import_timout: # input_json = {"type": "text", "tags": self.item_tags, "text": self.item_content}
item_not_imported = False # req = self.client.post('/api/v1/import/item', json=input_json, headers={'Authorization': self.apikey})
self.fail("Import error: {}".format(req_json)) # req_json = parse_response(self, req)
else: # import_uuid = req_json['uuid']
if time.time() - start > import_timout: # self.__class__.import_uuid = import_uuid
item_not_imported = False # self.assertTrue(Import_helper.is_valid_uuid_v4(import_uuid))
self.fail("Import Timeout, import status: {}".format(req_json['status'])) #
self.__class__.item_id = item_id # # POST /api/v1/get/import/item
# def test_0002_api_get_import_item(self):
# Process item # input_json = {"uuid": self.__class__.import_uuid}
time.sleep(5) # item_not_imported = True
# import_timout = 60
# POST /api/v1/get/item/content # start = time.time()
def test_0003_api_get_item_content(self): #
input_json = {"id": self.__class__.item_id} # while item_not_imported:
req = self.client.post('/api/v1/get/item/content', json=input_json ,headers={ 'Authorization': self.apikey }) # req = self.client.post('/api/v1/get/import/item', json=input_json, headers={'Authorization': self.apikey})
req_json = parse_response(self, req) # req_json = parse_response(self, req)
item_content = req_json['content'] # if req_json['status'] == 'imported':
self.assertEqual(item_content, self.item_content) # try:
# item_id = req_json['items'][0]
# POST /api/v1/get/item/tag # item_not_imported = False
def test_0004_api_get_item_tag(self): # except Exception as e:
input_json = {"id": self.__class__.item_id} # if time.time() - start > import_timout:
req = self.client.post('/api/v1/get/item/tag', json=input_json ,headers={ 'Authorization': self.apikey }) # item_not_imported = False
req_json = parse_response(self, req) # self.fail("Import error: {}".format(req_json))
item_tags = req_json['tags'] # else:
self.assertCountEqual(item_tags, self.expected_tags) # if time.time() - start > import_timout:
# item_not_imported = False
# POST /api/v1/get/item/tag # self.fail("Import Timeout, import status: {}".format(req_json['status']))
def test_0005_api_get_item_default(self): # self.__class__.item_id = item_id
input_json = {"id": self.__class__.item_id} #
req = self.client.post('/api/v1/get/item/default', json=input_json ,headers={ 'Authorization': self.apikey }) # # Process item
req_json = parse_response(self, req) # time.sleep(5)
item_tags = req_json['tags'] #
self.assertCountEqual(item_tags, self.expected_tags) # # POST /api/v1/get/item/content
item_content = req_json['content'] # def test_0003_api_get_item_content(self):
self.assertEqual(item_content, self.item_content) # input_json = {"id": self.__class__.item_id}
# req = self.client.post('/api/v1/get/item/content', json=input_json, headers={'Authorization': self.apikey})
# POST /api/v1/get/item/tag # req_json = parse_response(self, req)
# # TODO: add more test # item_content = req_json['content']
def test_0006_api_get_item(self): # self.assertEqual(item_content, self.item_content)
input_json = {"id": self.__class__.item_id, "content": True} #
req = self.client.post('/api/v1/get/item', json=input_json ,headers={ 'Authorization': self.apikey }) # # POST /api/v1/get/item/tag
req_json = parse_response(self, req) # def test_0004_api_get_item_tag(self):
item_tags = req_json['tags'] # input_json = {"id": self.__class__.item_id}
self.assertCountEqual(item_tags, self.expected_tags) # req = self.client.post('/api/v1/get/item/tag', json=input_json, headers={'Authorization': self.apikey})
item_content = req_json['content'] # req_json = parse_response(self, req)
self.assertEqual(item_content, self.item_content) # item_tags = req_json['tags']
# self.assertCountEqual(item_tags, self.expected_tags)
# POST api/v1/add/item/tag #
def test_0007_api_add_item_tag(self): # # POST /api/v1/get/item/tag
tags_to_add = ["infoleak:analyst-detection=\"api-key\""] # def test_0005_api_get_item_default(self):
current_item_tag = Tag.get_obj_tag(self.__class__.item_id) # input_json = {"id": self.__class__.item_id}
current_item_tag.append(tags_to_add[0]) # req = self.client.post('/api/v1/get/item/default', json=input_json, headers={'Authorization': self.apikey})
# req_json = parse_response(self, req)
#galaxy_to_add = ["misp-galaxy:stealer=\"Vidar\""] # item_tags = req_json['tags']
input_json = {"id": self.__class__.item_id, "tags": tags_to_add} # self.assertCountEqual(item_tags, self.expected_tags)
req = self.client.post('/api/v1/add/item/tag', json=input_json ,headers={ 'Authorization': self.apikey }) # item_content = req_json['content']
req_json = parse_response(self, req) # self.assertEqual(item_content, self.item_content)
item_tags = req_json['tags'] #
self.assertEqual(item_tags, tags_to_add) # # POST /api/v1/get/item/tag
# # # TODO: add more test
new_item_tag = Tag.get_obj_tag(self.__class__.item_id) # def test_0006_api_get_item(self):
self.assertCountEqual(new_item_tag, current_item_tag) # input_json = {"id": self.__class__.item_id, "content": True}
# req = self.client.post('/api/v1/get/item', json=input_json, headers={'Authorization': self.apikey})
# DELETE api/v1/delete/item/tag # req_json = parse_response(self, req)
def test_0008_api_add_item_tag(self): # item_tags = req_json['tags']
tags_to_delete = ["infoleak:analyst-detection=\"api-key\""] # self.assertCountEqual(item_tags, self.expected_tags)
input_json = {"id": self.__class__.item_id, "tags": tags_to_delete} # item_content = req_json['content']
req = self.client.delete('/api/v1/delete/item/tag', json=input_json ,headers={ 'Authorization': self.apikey }) # self.assertEqual(item_content, self.item_content)
req_json = parse_response(self, req) #
item_tags = req_json['tags'] # # POST api/v1/add/item/tag
self.assertCountEqual(item_tags, tags_to_delete) # def test_0007_api_add_item_tag(self):
current_item_tag = Tag.get_obj_tag(self.__class__.item_id) # tags_to_add = ["infoleak:analyst-detection=\"api-key\""]
if tags_to_delete[0] in current_item_tag: # current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
self.fail('Tag no deleted') # current_item_tag.append(tags_to_add[0])
#
# POST api/v1/get/tag/metadata # # galaxy_to_add = ["misp-galaxy:stealer=\"Vidar\""]
def test_0009_api_add_item_tag(self): # input_json = {"id": self.__class__.item_id, "tags": tags_to_add}
input_json = {"tag": self.item_tags[0]} # req = self.client.post('/api/v1/add/item/tag', json=input_json, headers={'Authorization': self.apikey})
req = self.client.post('/api/v1/get/tag/metadata', json=input_json ,headers={ 'Authorization': self.apikey }) # req_json = parse_response(self, req)
req_json = parse_response(self, req) # item_tags = req_json['tags']
self.assertEqual(req_json['tag'], self.item_tags[0]) # self.assertEqual(item_tags, tags_to_add)
#
# GET api/v1/get/tag/all # new_item_tag = Tag.get_obj_tag(self.__class__.item_id)
def test_0010_api_add_item_tag(self): # self.assertCountEqual(new_item_tag, current_item_tag)
input_json = {"tag": self.item_tags[0]} #
req = self.client.get('/api/v1/get/tag/all', json=input_json ,headers={ 'Authorization': self.apikey }) # # DELETE api/v1/delete/item/tag
req_json = parse_response(self, req) # def test_0008_api_add_item_tag(self):
self.assertTrue(req_json['tags']) # tags_to_delete = ["infoleak:analyst-detection=\"api-key\""]
# input_json = {"id": self.__class__.item_id, "tags": tags_to_delete}
# req = self.client.delete('/api/v1/delete/item/tag', json=input_json, headers={'Authorization': self.apikey})
# req_json = parse_response(self, req)
# item_tags = req_json['tags']
# self.assertCountEqual(item_tags, tags_to_delete)
# current_item_tag = Tag.get_obj_tag(self.__class__.item_id)
# if tags_to_delete[0] in current_item_tag:
# self.fail('Tag no deleted')
#
# # POST api/v1/get/tag/metadata
# def test_0009_api_add_item_tag(self):
# input_json = {"tag": self.item_tags[0]}
# req = self.client.post('/api/v1/get/tag/metadata', json=input_json, headers={'Authorization': self.apikey})
# req_json = parse_response(self, req)
# self.assertEqual(req_json['tag'], self.item_tags[0])
#
# # GET api/v1/get/tag/all
# def test_0010_api_add_item_tag(self):
# input_json = {"tag": self.item_tags[0]}
# req = self.client.get('/api/v1/get/tag/all', json=input_json, headers={'Authorization': self.apikey})
# req_json = parse_response(self, req)
# self.assertTrue(req_json['tags'])
#
#
if __name__ == "__main__": if __name__ == "__main__":
unittest.main(argv=['first-arg-is-ignored'], exit=False) unittest.main(argv=['first-arg-is-ignored'], exit=False)

22
tests/testHelper.py
View File

@ -1,22 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import unittest
import sys,os
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
class TestHelper(unittest.TestCase):
def setUp(self):
config_section = 'Keys'
def test_Process_Constructor_using_key_module(self):
conf_section = 'Keys'
process = Process(conf_section)
self.assertEqual(process.subscriber_name, 'Keys')

23
tests/test_modules.py
View File

@ -49,9 +49,9 @@ class Test_Module_ApiKey(unittest.TestCase):
aws_secret_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' aws_secret_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
matches = self.module_obj.compute(f'{item_id} 3', r_result=True) matches = self.module_obj.compute(f'{item_id} 3', r_result=True)
self.assertCountEqual(matches[0], [google_api_key]) self.assertCountEqual(matches[0], {google_api_key})
self.assertCountEqual(matches[1], [aws_access_key]) self.assertCountEqual(matches[1], {aws_access_key})
self.assertCountEqual(matches[2], [aws_secret_key]) self.assertCountEqual(matches[2], {aws_secret_key})
class Test_Module_Categ(unittest.TestCase): class Test_Module_Categ(unittest.TestCase):
@ -160,25 +160,8 @@ class Test_Module_Onion(unittest.TestCase):
item_id = 'tests/2021/01/01/onion.gz' item_id = 'tests/2021/01/01/onion.gz'
domain_1 = 'eswpccgr5xyovsahffkehgleqthrasfpfdblwbs4lstd345dwq5qumqd.onion' domain_1 = 'eswpccgr5xyovsahffkehgleqthrasfpfdblwbs4lstd345dwq5qumqd.onion'
domain_2 = 'www.facebookcorewwwi.onion' domain_2 = 'www.facebookcorewwwi.onion'
crawlers.queue_test_clean_up('onion', domain_1, 'tests/2021/01/01/onion.gz')
self.module_obj.compute(f'{item_id} 3') self.module_obj.compute(f'{item_id} 3')
if crawlers.is_crawler_activated():
# # check domain queues
# all domains queue
self.assertTrue(crawlers.is_domain_in_queue('onion', domain_1))
# all url/item queue
self.assertTrue(crawlers.is_item_in_queue('onion', f'http://{domain_1}', item_id))
# domain blacklist
self.assertFalse(crawlers.is_domain_in_queue('onion', domain_2))
# invalid onion
self.assertFalse(crawlers.is_domain_in_queue('onion', 'invalid.onion'))
# clean DB
crawlers.queue_test_clean_up('onion', domain_1, 'tests/2021/01/01/onion.gz')
else:
# # TODO: check warning logs
pass
class Test_Module_Telegram(unittest.TestCase): class Test_Module_Telegram(unittest.TestCase):