From ec7d2d49b27c75bd2a12075310918b675a37866c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 15 Nov 2017 16:03:42 +0100 Subject: [PATCH 01/12] Added seconds args in import_dir --- bin/import_dir.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/import_dir.py b/bin/import_dir.py index 1300cb43..3d291db0 100755 --- a/bin/import_dir.py +++ b/bin/import_dir.py @@ -53,6 +53,7 @@ if __name__ == "__main__": parser.add_argument('-p', '--port', type=int, default=5556, help='Zero MQ port') parser.add_argument('-c', '--channel', type=str, default='102', help='Zero MQ channel') parser.add_argument('-n', '--name', type=str, default='import_dir', help='Name of the feeder') + parser.add_argument('-s', '--seconds', type=float, default=0.2, help='Second between pastes') parser.add_argument('--hierarchy', type=int, default=1, help='Number of parent directory forming the name') args = parser.parse_args() @@ -90,4 +91,4 @@ if __name__ == "__main__": print(args.name+'>'+wanted_path) path_to_send = args.name + '>' + wanted_path socket.send('{} {} {}'.format(args.channel, path_to_send, base64.b64encode(messagedata))) - time.sleep(.2) + time.sleep(args.seconds) From 652b00f4b28b2d75fb0d258e28f99b03226febb8 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 15 Nov 2017 16:15:43 +0100 Subject: [PATCH 02/12] Renamed BrowseWarningPaste into alertHandler --- bin/Credential.py | 4 ++-- bin/CreditCards.py | 2 +- bin/Cve.py | 2 +- bin/Keys.py | 2 +- bin/LAUNCH.sh | 2 +- bin/Mail.py | 2 +- bin/Onion.py | 2 +- bin/Phone.py | 2 +- bin/SQLInjectionDetection.py | 2 +- ...{BrowseWarningPaste.py => alertHandler.py} | 8 +++---- bin/launch_scripts.sh | 2 +- bin/packages/modules.cfg | 24 +++++++++---------- 12 files changed, 26 insertions(+), 28 deletions(-) rename bin/{BrowseWarningPaste.py => alertHandler.py} (85%) diff --git a/bin/Credential.py b/bin/Credential.py index 233e424a..bb52f311 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -99,8 +99,8 @@ if __name__ == "__main__": publisher.warning(to_print) #Send to duplicate p.populate_set_out(filepath, 'Duplicate') - #Send to BrowseWarningPaste - p.populate_set_out('credential;{}'.format(filepath), 'BrowseWarningPaste') + #Send to alertHandler + p.populate_set_out('credential;{}'.format(filepath), 'alertHandler') #Put in form, count occurences, then send to moduleStats creds_sites = {} diff --git a/bin/CreditCards.py b/bin/CreditCards.py index 79442576..133916fe 100755 --- a/bin/CreditCards.py +++ b/bin/CreditCards.py @@ -79,7 +79,7 @@ if __name__ == "__main__": #Send to duplicate p.populate_set_out(filename, 'Duplicate') #send to Browse_warning_paste - p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste') + p.populate_set_out('creditcard;{}'.format(filename), 'alertHandler') else: publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path)) else: diff --git a/bin/Cve.py b/bin/Cve.py index fb4b0b24..62df0aba 100755 --- a/bin/Cve.py +++ b/bin/Cve.py @@ -32,7 +32,7 @@ def search_cve(message): publisher.warning('{} contains CVEs'.format(paste.p_name)) #send to Browse_warning_paste - p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste') + p.populate_set_out('cve;{}'.format(filepath), 'alertHandler') #Send to duplicate p.populate_set_out(filepath, 'Duplicate') diff --git a/bin/Keys.py b/bin/Keys.py index d2e7ebd2..61d52602 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -26,7 +26,7 @@ def search_gpg(message): #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste - p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste') + p.populate_set_out('keys;{}'.format(message), 'alertHandler') if __name__ == '__main__': diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 227d3a9b..fb779182 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -170,7 +170,7 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' + screen -S "Script" -X screen -t "alertHandler" bash -c './alertHandler.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' diff --git a/bin/Mail.py b/bin/Mail.py index 99dd6948..c608d106 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -72,7 +72,7 @@ if __name__ == "__main__": publisher.warning(to_print) #Send to duplicate p.populate_set_out(filename, 'Duplicate') - p.populate_set_out('mail;{}'.format(filename), 'BrowseWarningPaste') + p.populate_set_out('mail;{}'.format(filename), 'alertHandler') else: publisher.info(to_print) diff --git a/bin/Onion.py b/bin/Onion.py index af41777d..aaf30a1b 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -145,7 +145,7 @@ if __name__ == "__main__": PST.p_name) for url in fetch(p, r_cache, urls, domains_list, path): publisher.warning('{}Checked {};{}'.format(to_print, url, PST.p_path)) - p.populate_set_out('onion;{}'.format(PST.p_path), 'BrowseWarningPaste') + p.populate_set_out('onion;{}'.format(PST.p_path), 'alertHandler') else: publisher.info('{}Onion related;{}'.format(to_print, PST.p_path)) diff --git a/bin/Phone.py b/bin/Phone.py index cb32a691..7a4811da 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -33,7 +33,7 @@ def search_phone(message): print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) #send to Browse_warning_paste - p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') + p.populate_set_out('phone;{}'.format(message), 'alertHandler') #Send to duplicate p.populate_set_out(message, 'Duplicate') stats = {} diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index d2948f1b..318466c8 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -81,7 +81,7 @@ def analyse(url, path): #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste - p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste') + p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') else: print "Potential SQL injection:" print urllib2.unquote(url) diff --git a/bin/BrowseWarningPaste.py b/bin/alertHandler.py similarity index 85% rename from bin/BrowseWarningPaste.py rename to bin/alertHandler.py index 4f49f56b..ba6bb4d4 100755 --- a/bin/BrowseWarningPaste.py +++ b/bin/alertHandler.py @@ -24,7 +24,7 @@ if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" - config_section = 'BrowseWarningPaste' + config_section = 'alertHandler' p = Process(config_section) @@ -48,12 +48,10 @@ if __name__ == "__main__": time.sleep(10) continue - # Add in redis + # Add in redis for browseWarningPaste # Format in set: WARNING_moduleName -> p_path key = "WARNING_" + module_name - print key + ' -> ' + p_path server.sadd(key, p_path) - publisher.info('Saved in warning paste {}'.format(p_path)) - #print 'Saved in warning paste {}'.format(p_path) + publisher.info('Saved warning paste {}'.format(p_path)) diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index e593b11e..0dd29c2f 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -72,6 +72,6 @@ screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' sleep 0.1 -screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' +screen -S "Script" -X screen -t "alertHandler" bash -c './alertHandler.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 33eebd21..55fb46d4 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -49,16 +49,16 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re [CreditCards] subscribe = Redis_CreditCards -publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste +publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler [Mail] subscribe = Redis_Mail -publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste +publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler [Onion] subscribe = Redis_Onion -publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_BrowseWarningPaste -#publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion,Redis_BrowseWarningPaste +publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler +#publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler [DumpValidOnion] subscribe = Redis_ValidOnion @@ -72,17 +72,17 @@ subscribe = Redis_Url [SQLInjectionDetection] subscribe = Redis_Url -publish = Redis_BrowseWarningPaste,Redis_Duplicate +publish = Redis_alertHandler,Redis_Duplicate [ModuleStats] subscribe = Redis_ModuleStats -[BrowseWarningPaste] -subscribe = Redis_BrowseWarningPaste +[alertHandler] +subscribe = Redis_alertHandler #[send_to_queue] #subscribe = Redis_Cve -#publish = Redis_BrowseWarningPaste +#publish = Redis_alertHandler [SentimentAnalysis] subscribe = Redis_Global @@ -92,16 +92,16 @@ subscribe = Redis_Global [Credential] subscribe = Redis_Credential -publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste +publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler [Cve] subscribe = Redis_Cve -publish = Redis_BrowseWarningPaste,Redis_Duplicate +publish = Redis_alertHandler,Redis_Duplicate [Phone] subscribe = Redis_Global -publish = Redis_Duplicate,Redis_BrowseWarningPaste +publish = Redis_Duplicate,Redis_alertHandler [Keys] subscribe = Redis_Global -publish = Redis_Duplicate,Redis_BrowseWarningPaste +publish = Redis_Duplicate,Redis_alertHandler From 45a989702037ad1475af9c7d2af5ef6842757ef1 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 16 Nov 2017 09:52:37 +0100 Subject: [PATCH 03/12] Added draft support of MISP ail-leak object --- bin/ailleakOject.py | 25 +++++++++++++++++++++++++ bin/alertHandler.py | 20 ++++++++++++++++++++ mispKEYS.py.default | 6 ++++++ 3 files changed, 51 insertions(+) create mode 100644 bin/ailleakOject.py create mode 100644 mispKEYS.py.default diff --git a/bin/ailleakOject.py b/bin/ailleakOject.py new file mode 100644 index 00000000..03cb6c7f --- /dev/null +++ b/bin/ailleakOject.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +from pymisp.tools import GenericObjectGenerator +from packages import Paste + +class AilleakObject(AbstractMISPObjectGenerator): + def __init__(self, moduleName, path): + super(GenericObject, self).__init__('ail-leak') + self.moduleName = moduleName + self.path = path + self.paste = Paste.Paste(path) + self.generate_attributes() + + def generate_attributes(self): + self.add_attribute('type', value=self.moduleName) + self.add_attribute('origin', value=self.paste.p_source) + self.add_attribute('last-seen', value=self.paste.p_date) + self.add_attribute('raw-data', value=self.paste.get_p_content()) + ''' + # duplicated + duplicate_list = json.loads(paste._get_p_duplicate()) + is_duplicate = True if len(duplicate_list) > 0 else False + self.add_attribute('duplicate', value=is_duplicate) + ''' diff --git a/bin/alertHandler.py b/bin/alertHandler.py index ba6bb4d4..cb654639 100755 --- a/bin/alertHandler.py +++ b/bin/alertHandler.py @@ -20,6 +20,10 @@ from packages import Paste from pubsublogger import publisher from Helper import Process +from pymisp import PyMISP +import ailleakObject +from ../mispKEYS import misp_url, misp_key, misp_verifycert + if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" @@ -27,6 +31,9 @@ if __name__ == "__main__": config_section = 'alertHandler' p = Process(config_section) + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + eventID = "9356" + mispTYPE = 'ail-leak' # port generated automatically depending on the date curYear = datetime.now().year @@ -55,3 +62,16 @@ if __name__ == "__main__": publisher.info('Saved warning paste {}'.format(p_path)) + # Create MISP AIL-leak object + misp_object = AilleakObject(moduleName, path) + print('validate mispobj', misp_object._validate()) + print(misp_object) + + # Publish object to MISP + try: + templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE + except IndexError: + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) + continue + #r = pymisp.add_object(eventID, templateID, misp_object) diff --git a/mispKEYS.py.default b/mispKEYS.py.default new file mode 100644 index 00000000..42c534b8 --- /dev/null +++ b/mispKEYS.py.default @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +misp_url = '' +misp_key = '' # The MISP auth key can be found on the MISP web interface under the automation section +misp_verifycert = True From ee7759be91d0e1c9ae9b696e74263c07519ac4d0 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 16 Nov 2017 11:18:13 +0100 Subject: [PATCH 04/12] Updated alertHandler and ailleakObject --- bin/ailleakOject.py | 36 ++++++++++++++++++++++++++++++++---- bin/alertHandler.py | 14 ++++++++------ 2 files changed, 40 insertions(+), 10 deletions(-) mode change 100644 => 100755 bin/ailleakOject.py diff --git a/bin/ailleakOject.py b/bin/ailleakOject.py old mode 100644 new mode 100755 index 03cb6c7f..d8475779 --- a/bin/ailleakOject.py +++ b/bin/ailleakOject.py @@ -1,12 +1,12 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.5 # -*-coding:UTF-8 -* -from pymisp.tools import GenericObjectGenerator +from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator from packages import Paste class AilleakObject(AbstractMISPObjectGenerator): def __init__(self, moduleName, path): - super(GenericObject, self).__init__('ail-leak') + super(AbstractMISPObjectGenerator, self).__init__('ail-leak') self.moduleName = moduleName self.path = path self.paste = Paste.Paste(path) @@ -16,10 +16,38 @@ class AilleakObject(AbstractMISPObjectGenerator): self.add_attribute('type', value=self.moduleName) self.add_attribute('origin', value=self.paste.p_source) self.add_attribute('last-seen', value=self.paste.p_date) - self.add_attribute('raw-data', value=self.paste.get_p_content()) + #self.add_attribute('raw-data', value=self.paste.get_p_content()) ''' # duplicated duplicate_list = json.loads(paste._get_p_duplicate()) is_duplicate = True if len(duplicate_list) > 0 else False self.add_attribute('duplicate', value=is_duplicate) ''' + + +if __name__ == "__main__": + + import sys + sys.path.append('../') + from mispKEYS import misp_url, misp_key, misp_verifycert + from pymisp import PyMISP + + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + eventID = "9356" + mispTYPE = 'ail-leak' + + moduleName = "Credentials" + path = "/home/sami/git/AIL-framework/PASTES/archive/pastebin.com_pro/2017/08/23/bPFaJymf.gz" + + misp_object = AilleakObject(moduleName, path) + print('validate mispobj', misp_object._validate()) + print(misp_object) + + # Publish object to MISP + try: + templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] + except IndexError: + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) + print(templateID) + #r = pymisp.add_object(eventID, templateID, misp_object) diff --git a/bin/alertHandler.py b/bin/alertHandler.py index cb654639..d5f23545 100755 --- a/bin/alertHandler.py +++ b/bin/alertHandler.py @@ -22,7 +22,9 @@ from Helper import Process from pymisp import PyMISP import ailleakObject -from ../mispKEYS import misp_url, misp_key, misp_verifycert +import sys +sys.path.append('../') +from mispKEYS import misp_url, misp_key, misp_verifycert if __name__ == "__main__": publisher.port = 6380 @@ -69,9 +71,9 @@ if __name__ == "__main__": # Publish object to MISP try: - templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE - except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) - print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - continue + templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] + except IndexError: + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) + continue #r = pymisp.add_object(eventID, templateID, misp_object) From 6f0227a78a971ea5576fc1278743a2a7e532c5d2 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 20 Nov 2017 12:12:06 +0100 Subject: [PATCH 05/12] Starting support of python3 --- bin/ailleakOject.py | 16 ++-------------- bin/packages/Date.py | 8 ++++---- bin/packages/Paste.py | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/bin/ailleakOject.py b/bin/ailleakOject.py index d8475779..8459a126 100755 --- a/bin/ailleakOject.py +++ b/bin/ailleakOject.py @@ -33,21 +33,9 @@ if __name__ == "__main__": from pymisp import PyMISP pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - eventID = "9356" - mispTYPE = 'ail-leak' moduleName = "Credentials" path = "/home/sami/git/AIL-framework/PASTES/archive/pastebin.com_pro/2017/08/23/bPFaJymf.gz" - misp_object = AilleakObject(moduleName, path) - print('validate mispobj', misp_object._validate()) - print(misp_object) - - # Publish object to MISP - try: - templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] - except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) - print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - print(templateID) - #r = pymisp.add_object(eventID, templateID, misp_object) + wrapper = objectWrapper(moduleName, path, pymisp) + wrapper.pushToMISP() diff --git a/bin/packages/Date.py b/bin/packages/Date.py index ce02636a..85da5b36 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -32,10 +32,10 @@ class Date(object): self.day = day def substract_day(self, numDay): - import datetime - computed_date = datetime.date(int(self.year), int(self.month), int(self.day)) - datetime.timedelta(numDay) - comp_year = str(computed_date.year) + import datetime + computed_date = datetime.date(int(self.year), int(self.month), int(self.day)) - datetime.timedelta(numDay) + comp_year = str(computed_date.year) comp_month = str(computed_date.month).zfill(2) comp_day = str(computed_date.day).zfill(2) - return comp_year + comp_month + comp_day + return comp_year + comp_month + comp_day diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index 6d18e846..1debd33e 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -24,8 +24,17 @@ import operator import string import re import json -import ConfigParser -import cStringIO +try: # dirty to support python3 + import ConfigParser +except: + import configparser + ConfigParser = configparser +try: # dirty to support python3 + import cStringIO +except: + from io import StringIO as cStringIO +import sys +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) from Date import Date from Hash import Hash @@ -84,6 +93,7 @@ class Paste(object): var = self.p_path.split('/') self.p_date = Date(var[-4], var[-3], var[-2]) self.p_source = var[-5] + self.supposed_url = 'https://{}/{}'.format(self.p_source.replace('_pro', ''), var[-1].split('.gz')[0]) self.p_encoding = None self.p_hash_kind = {} From 1e76e10c012a9c84b6c29c8a4e263c58639f26a3 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 20 Nov 2017 12:12:53 +0100 Subject: [PATCH 06/12] Support of add_object to a MISP instance --- bin/ailleakOject.py | 89 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/bin/ailleakOject.py b/bin/ailleakOject.py index 8459a126..d56b5012 100755 --- a/bin/ailleakOject.py +++ b/bin/ailleakOject.py @@ -3,20 +3,38 @@ from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator from packages import Paste +import datetime class AilleakObject(AbstractMISPObjectGenerator): - def __init__(self, moduleName, path): + def __init__(self, moduleName, p_source, p_date): + #def __init__(self, moduleName, p_source, p_date, p_content): super(AbstractMISPObjectGenerator, self).__init__('ail-leak') self.moduleName = moduleName - self.path = path - self.paste = Paste.Paste(path) + self.p_source = p_source + self.p_date = p_date + #self.p_content = p_content self.generate_attributes() def generate_attributes(self): self.add_attribute('type', value=self.moduleName) - self.add_attribute('origin', value=self.paste.p_source) - self.add_attribute('last-seen', value=self.paste.p_date) - #self.add_attribute('raw-data', value=self.paste.get_p_content()) + self.add_attribute('origin', value=self.p_source) + self.add_attribute('last-seen', value=self.p_date) + #self.add_attribute('raw-data', value=self.p_content) + +class objectWrapper: + def __init__(self, moduleName, path, pymisp): + self.moduleName = moduleName + self.path = path + self.pymisp = pymisp + self.paste = Paste.Paste(path) + self.p_date = self.date_to_str(self.paste.p_date) + self.p_source = self.paste.supposed_url + self.p_content = self.paste.get_p_content() + + self.eventID_to_push = self.get_daily_event_id() + self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date) + #self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content) + ''' # duplicated duplicate_list = json.loads(paste._get_p_duplicate()) @@ -24,6 +42,65 @@ class AilleakObject(AbstractMISPObjectGenerator): self.add_attribute('duplicate', value=is_duplicate) ''' + def date_to_str(self, date): + return "{0}-{1}-{2}".format(date.year, date.month, date.day) + + def get_all_related_events(self): + to_search = "Daily AIL-leaks" + result = pymisp.search_all(to_search) + events = [] + for e in result['response']: + events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) + return events + + def get_daily_event_id(self): + to_match = "Daily AIL-leaks {}".format(datetime.date.today()) + events = self.get_all_related_events() + for dic in events: + info = dic['info'] + e_id = dic['id'] + if info == to_match: + print('Found: ', info, '->', e_id) + return e_id + created_event = self.create_daily_event()['Event'] + new_id = created_event['id'] + print('New event created:', new_id) + return new_id + + + def create_daily_event(self): + today = datetime.date.today() + # [0-3] + distribution = 0 + info = "Daily AIL-leaks {}".format(today) + # [0-2] + analysis = 0 + # [1-4] + threat = 3 + published = False + org_id = None + orgc_id = None + sharing_group_id = None + date = None + event = self.pymisp.new_event(distribution, threat, + analysis, info, date, + published, orgc_id, org_id, sharing_group_id) + return event + + # Publish object to MISP + def pushToMISP(self): + mispTYPE = 'ail-leak' + try: + templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] + except IndexError: + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) + r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject) + if 'errors' in r: + print(r) + else: + print('Pushed:', self.moduleName, '->', self.p_source) + if __name__ == "__main__": From e0b188c0694d4b66c812a913736a0a9c0ecc14b5 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 20 Nov 2017 14:55:36 +0100 Subject: [PATCH 07/12] Better support of python3 in Helper.py --- bin/Helper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/Helper.py b/bin/Helper.py index 7d3abcbd..2560f340 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -12,7 +12,11 @@ the same Subscriber name in both of them. """ import redis -import ConfigParser +try: # dirty to support python3 + import ConfigParser +except: + import configparser + ConfigParser = configparser import os import zmq import time From 805171a8a5361ecc58abc1fe49c529b0c43b2e50 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 20 Nov 2017 14:57:25 +0100 Subject: [PATCH 08/12] Added a seemingly working version of ailleak to misp --- bin/{ailleakOject.py => ailleakObject.py} | 61 ++++++++++++++++------- bin/alertHandler.py | 27 ++++------ 2 files changed, 55 insertions(+), 33 deletions(-) rename bin/{ailleakOject.py => ailleakObject.py} (69%) diff --git a/bin/ailleakOject.py b/bin/ailleakObject.py similarity index 69% rename from bin/ailleakOject.py rename to bin/ailleakObject.py index d56b5012..d2de82bf 100755 --- a/bin/ailleakOject.py +++ b/bin/ailleakObject.py @@ -4,36 +4,56 @@ from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator from packages import Paste import datetime +import json class AilleakObject(AbstractMISPObjectGenerator): - def __init__(self, moduleName, p_source, p_date): - #def __init__(self, moduleName, p_source, p_date, p_content): + def __init__(self, moduleName, p_source, p_date, p_content, p_duplicate): super(AbstractMISPObjectGenerator, self).__init__('ail-leak') self.moduleName = moduleName self.p_source = p_source self.p_date = p_date - #self.p_content = p_content + self.p_content = p_content + self.p_duplicate = p_duplicate self.generate_attributes() def generate_attributes(self): self.add_attribute('type', value=self.moduleName) self.add_attribute('origin', value=self.p_source) self.add_attribute('last-seen', value=self.p_date) - #self.add_attribute('raw-data', value=self.p_content) + self.add_attribute('duplicate-list', value=self.p_duplicate) + self.add_attribute('raw-data', value=self.p_content) -class objectWrapper: - def __init__(self, moduleName, path, pymisp): +class ObjectWrapper: + def __init__(self, pymisp): + self.pymisp = pymisp + self.currentID_date = None + self.eventID_to_push = self.get_daily_event_id() + + def add_new_object(self, moduleName, path): self.moduleName = moduleName self.path = path - self.pymisp = pymisp self.paste = Paste.Paste(path) self.p_date = self.date_to_str(self.paste.p_date) self.p_source = self.paste.supposed_url - self.p_content = self.paste.get_p_content() - - self.eventID_to_push = self.get_daily_event_id() - self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date) - #self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content) + self.p_content = self.paste.get_p_content().decode('utf8') + + temp = self.paste._get_p_duplicate() + try: + temp = temp.decode('utf8') + except AttributeError: + print('decode error') + #beautifier + temp = json.loads(temp) + to_ret = [] + for dup in temp: + algo = dup[0] + path = dup[1].split('/')[-5:] + perc = dup[2] + to_ret.append([path, algo, perc]) + self.p_duplicate = str(to_ret) + + + self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate) ''' # duplicated @@ -42,12 +62,13 @@ class objectWrapper: self.add_attribute('duplicate', value=is_duplicate) ''' + def date_to_str(self, date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) def get_all_related_events(self): to_search = "Daily AIL-leaks" - result = pymisp.search_all(to_search) + result = self.pymisp.search_all(to_search) events = [] for e in result['response']: events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) @@ -61,10 +82,12 @@ class objectWrapper: e_id = dic['id'] if info == to_match: print('Found: ', info, '->', e_id) + self.currentID_date = datetime.date.today() return e_id created_event = self.create_daily_event()['Event'] new_id = created_event['id'] print('New event created:', new_id) + self.currentID_date = datetime.date.today() return new_id @@ -89,11 +112,14 @@ class objectWrapper: # Publish object to MISP def pushToMISP(self): + if self.currentID_date != datetime.date.today(): #refresh id + self.eventID_to_push = self.get_daily_event_id() + mispTYPE = 'ail-leak' try: - templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] + templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()]) print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject) if 'errors' in r: @@ -101,7 +127,7 @@ class objectWrapper: else: print('Pushed:', self.moduleName, '->', self.p_source) - +''' if __name__ == "__main__": import sys @@ -114,5 +140,6 @@ if __name__ == "__main__": moduleName = "Credentials" path = "/home/sami/git/AIL-framework/PASTES/archive/pastebin.com_pro/2017/08/23/bPFaJymf.gz" - wrapper = objectWrapper(moduleName, path, pymisp) + wrapper = ObjectWrapper(moduleName, path, pymisp) wrapper.pushToMISP() +''' diff --git a/bin/alertHandler.py b/bin/alertHandler.py index d5f23545..299f4961 100755 --- a/bin/alertHandler.py +++ b/bin/alertHandler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3.5 # -*-coding:UTF-8 -* """ @@ -34,8 +34,8 @@ if __name__ == "__main__": p = Process(config_section) pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - eventID = "9356" - mispTYPE = 'ail-leak' + print('Connected to MISP:', misp_url) + wrapper = ailleakObject.ObjectWrapper(pymisp) # port generated automatically depending on the date curYear = datetime.now().year @@ -50,6 +50,7 @@ if __name__ == "__main__": while True: message = p.get_from_set() if message is not None: + message = message.decode('utf8') #decode because of pyhton3 module_name, p_path = message.split(';') #PST = Paste.Paste(p_path) else: @@ -64,16 +65,10 @@ if __name__ == "__main__": publisher.info('Saved warning paste {}'.format(p_path)) - # Create MISP AIL-leak object - misp_object = AilleakObject(moduleName, path) - print('validate mispobj', misp_object._validate()) - print(misp_object) - - # Publish object to MISP - try: - templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] - except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) - print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - continue - #r = pymisp.add_object(eventID, templateID, misp_object) + # Create MISP AIL-leak object and push it + allowed_modules = ['credential'] + if module_name in allowed_modules: + wrapper.add_new_object(module_name, p_path) + wrapper.pushToMISP() + else: + print('not pushing to MISP:', module_name, p_path) From 2a967c4d9278fce20c515106f2897d6e107bce42 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Nov 2017 07:13:44 +0100 Subject: [PATCH 09/12] update/feature: Max number of duplicate push to MISP + duplicate are pushed as attachment --- bin/ailleakObject.py | 54 +++++++++++++++++++++------------- bin/alertHandler.py | 2 +- bin/packages/config.cfg.sample | 3 ++ 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/bin/ailleakObject.py b/bin/ailleakObject.py index d2de82bf..906b4ae0 100755 --- a/bin/ailleakObject.py +++ b/bin/ailleakObject.py @@ -2,58 +2,69 @@ # -*-coding:UTF-8 -* from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator +import configparser from packages import Paste import datetime import json +from io import BytesIO class AilleakObject(AbstractMISPObjectGenerator): - def __init__(self, moduleName, p_source, p_date, p_content, p_duplicate): + def __init__(self, moduleName, p_source, p_date, p_content, p_duplicate, p_duplicate_number): super(AbstractMISPObjectGenerator, self).__init__('ail-leak') - self.moduleName = moduleName - self.p_source = p_source - self.p_date = p_date - self.p_content = p_content - self.p_duplicate = p_duplicate + self._moduleName = moduleName + self._p_source = p_source.split('/')[-5:] + self._p_source = '/'.join(self._p_source)[:-3] # -3 removes .gz + self._p_date = p_date + self._p_content = p_content.encode('utf8') + self._p_duplicate = p_duplicate + self._p_duplicate_number = p_duplicate_number self.generate_attributes() def generate_attributes(self): - self.add_attribute('type', value=self.moduleName) - self.add_attribute('origin', value=self.p_source) - self.add_attribute('last-seen', value=self.p_date) - self.add_attribute('duplicate-list', value=self.p_duplicate) - self.add_attribute('raw-data', value=self.p_content) + self.add_attribute('type', value=self._moduleName) + self.add_attribute('origin', value=self._p_source, type='text') + self.add_attribute('last-seen', value=self._p_date) + if self._p_duplicate_number > 0: + self.add_attribute('duplicate', value=self._p_duplicate, type='text') + self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter') + self._pseudofile = BytesIO(self._p_content) + self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment") class ObjectWrapper: def __init__(self, pymisp): self.pymisp = pymisp self.currentID_date = None self.eventID_to_push = self.get_daily_event_id() + cfg = configparser.ConfigParser() + cfg.read('./packages/config.cfg') + self.maxDuplicateToPushToMISP = cfg.getint("ailleakObject", "maxDuplicateToPushToMISP") def add_new_object(self, moduleName, path): self.moduleName = moduleName self.path = path self.paste = Paste.Paste(path) self.p_date = self.date_to_str(self.paste.p_date) - self.p_source = self.paste.supposed_url + self.p_source = self.paste.p_path self.p_content = self.paste.get_p_content().decode('utf8') temp = self.paste._get_p_duplicate() try: temp = temp.decode('utf8') except AttributeError: - print('decode error') + pass #beautifier temp = json.loads(temp) - to_ret = [] - for dup in temp: + self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0 + to_ret = "" + for dup in temp[:self.maxDuplicateToPushToMISP]: algo = dup[0] path = dup[1].split('/')[-5:] + path = '/'.join(path)[:-3] # -3 removes .gz perc = dup[2] - to_ret.append([path, algo, perc]) - self.p_duplicate = str(to_ret) - + to_ret += "{}: {} [{}%]\n".format(path, algo, perc) + self.p_duplicate = to_ret - self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate) + self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number) ''' # duplicated @@ -137,9 +148,10 @@ if __name__ == "__main__": pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - moduleName = "Credentials" + moduleName = "credentials" path = "/home/sami/git/AIL-framework/PASTES/archive/pastebin.com_pro/2017/08/23/bPFaJymf.gz" - wrapper = ObjectWrapper(moduleName, path, pymisp) + wrapper = ObjectWrapper(pymisp) + wrapper.add_new_object(moduleName, path) wrapper.pushToMISP() ''' diff --git a/bin/alertHandler.py b/bin/alertHandler.py index 299f4961..32fb3731 100755 --- a/bin/alertHandler.py +++ b/bin/alertHandler.py @@ -66,7 +66,7 @@ if __name__ == "__main__": publisher.info('Saved warning paste {}'.format(p_path)) # Create MISP AIL-leak object and push it - allowed_modules = ['credential'] + allowed_modules = ['credential', 'phone', 'creditcards'] if module_name in allowed_modules: wrapper.add_new_object(module_name, p_path) wrapper.pushToMISP() diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 1adaf04d..da50932f 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -130,6 +130,9 @@ register = indexdir/all_index.txt #size in Mb index_max_size = 2000 +[ailleakObject] +maxDuplicateToPushToMISP=10 + ############################################################################### # For multiple feed, add them with "," without space From a0d07b1098030f3177fb1d0e08d148631d42002a Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Nov 2017 13:47:02 +0100 Subject: [PATCH 10/12] feature: try to send data to MISP if keys are provided, else do nothing --- bin/alertHandler.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/bin/alertHandler.py b/bin/alertHandler.py index 32fb3731..ce473ed4 100755 --- a/bin/alertHandler.py +++ b/bin/alertHandler.py @@ -24,7 +24,12 @@ from pymisp import PyMISP import ailleakObject import sys sys.path.append('../') -from mispKEYS import misp_url, misp_key, misp_verifycert +try: + from mispKEYS import misp_url, misp_key, misp_verifycert + flag_misp = True +except: + print('Misp keys not present') + flag_misp = False if __name__ == "__main__": publisher.port = 6380 @@ -33,9 +38,16 @@ if __name__ == "__main__": config_section = 'alertHandler' p = Process(config_section) - pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - print('Connected to MISP:', misp_url) - wrapper = ailleakObject.ObjectWrapper(pymisp) + if flag_misp: + try: + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + print('Connected to MISP:', misp_url) + except: + flag_misp = False + print('Not connected to MISP') + + if flag_misp: + wrapper = ailleakObject.ObjectWrapper(pymisp) # port generated automatically depending on the date curYear = datetime.now().year @@ -66,9 +78,10 @@ if __name__ == "__main__": publisher.info('Saved warning paste {}'.format(p_path)) # Create MISP AIL-leak object and push it - allowed_modules = ['credential', 'phone', 'creditcards'] - if module_name in allowed_modules: - wrapper.add_new_object(module_name, p_path) - wrapper.pushToMISP() - else: - print('not pushing to MISP:', module_name, p_path) + if flag_misp: + allowed_modules = ['credential', 'phone', 'creditcards'] + if module_name in allowed_modules: + wrapper.add_new_object(module_name, p_path) + wrapper.pushToMISP() + else: + print('not pushing to MISP:', module_name, p_path) From f603a7f44b290ddb88c1f6d783a7535ce991719a Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 23 Nov 2017 14:02:54 +0100 Subject: [PATCH 11/12] Added pip3 in dependencies --- installing_deps.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/installing_deps.sh b/installing_deps.sh index 9c414fe7..6c70488d 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -11,6 +11,10 @@ sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \ #Needed for bloom filters sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y +#pyMISP +sudo apt-get -y install python3-pip +sudo pip3 install pymisp + # DNS deps sudo apt-get install libadns1 libadns1-dev -y From 83e082e62a20a49e1ca2d546e4f19209135ac59d Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 24 Nov 2017 08:57:41 +0100 Subject: [PATCH 12/12] update: removed useless comments --- bin/ailleakObject.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/bin/ailleakObject.py b/bin/ailleakObject.py index 906b4ae0..8b7ea185 100755 --- a/bin/ailleakObject.py +++ b/bin/ailleakObject.py @@ -66,14 +66,6 @@ class ObjectWrapper: self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number) - ''' - # duplicated - duplicate_list = json.loads(paste._get_p_duplicate()) - is_duplicate = True if len(duplicate_list) > 0 else False - self.add_attribute('duplicate', value=is_duplicate) - ''' - - def date_to_str(self, date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) @@ -137,21 +129,3 @@ class ObjectWrapper: print(r) else: print('Pushed:', self.moduleName, '->', self.p_source) - -''' -if __name__ == "__main__": - - import sys - sys.path.append('../') - from mispKEYS import misp_url, misp_key, misp_verifycert - from pymisp import PyMISP - - pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - - moduleName = "credentials" - path = "/home/sami/git/AIL-framework/PASTES/archive/pastebin.com_pro/2017/08/23/bPFaJymf.gz" - - wrapper = ObjectWrapper(pymisp) - wrapper.add_new_object(moduleName, path) - wrapper.pushToMISP() -'''