From 370011c0817f8f773eed0f1406c2fea85ca94bf5 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 2 May 2018 12:43:34 +0200 Subject: [PATCH 1/5] threatanalyzer_import - fix regkey issue --- misp_modules/modules/import_mod/threatanalyzer_import.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index fded508..fd16246 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -457,8 +457,6 @@ def cleanup_regkey(item): r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\' } - item = item.replace('\\REGISTRY\\MACHINE\\', 'HKLM\\') - item = item.replace('\\REGISTRY\\USER\\', 'HKCU\\') if list_in_string(noise_substrings, item, regex=True): return None return item From 27a22e5d86ecb1b7c2f81cfa1f02401464270cd1 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Thu, 3 May 2018 09:42:38 +0200 Subject: [PATCH 2/5] threatanalyzer_import - loads sample info + pollution fix --- .../import_mod/threatanalyzer_import.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index fd16246..da01b93 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -62,12 +62,12 @@ def handler(q=False): if re.match(r"Analysis/proc_\d+/modified_files/.+\.", zip_file_name) and "mapping.log" not in zip_file_name: sample_md5 = zip_file_name.split('/')[-1].split('.')[0] if sample_md5 in modified_files_mapping: - sample_filename = modified_files_mapping[sample_md5] - # print("{} maps to {}".format(sample_md5, sample_filename)) + current_sample_filename = modified_files_mapping[sample_md5] + # print("{} maps to {}".format(sample_md5, current_sample_filename)) with zf.open(zip_file_name, mode='r', pwd=None) as fp: file_data = fp.read() results.append({ - 'values': sample_filename, + 'values': current_sample_filename, 'data': base64.b64encode(file_data).decode(), 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) @@ -76,8 +76,18 @@ def handler(q=False): file_data = fp.read() analysis_json = json.loads(file_data.decode('utf-8')) results += process_analysis_json(analysis_json) - # if 'sample' in zip_file_name: - # sample['data'] = base64.b64encode(file_data).decode() + try: + sample_filename = analysis_json.get('analysis').get('@filename') + if sample_filename: + with zf.open('sample', mode='r', pwd=None) as fp: + file_data = fp.read() + results.append({ + 'values': sample_filename, + 'data': base64.b64encode(file_data).decode(), + 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) + except Exception as e: + # no 'sample' in archive, might be an url analysis, just ignore + pass else: try: @@ -455,7 +465,9 @@ def cleanup_regkey(item): r'\\Software\\Classes\\CLSID\\', r'\\Software\\Classes\\Local Settings\\MuiCache\\', r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\' + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', + r'\\Software\\Microsoft\\Tracing\\powershell_RASMANCS\\', + r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\' } if list_in_string(noise_substrings, item, regex=True): return None From 7b4db1ce5ce99491a47b2cdb006a5bf80542e676 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 15 May 2018 12:59:55 +0200 Subject: [PATCH 3/5] threatanalyzer_import - minor generic noise removal --- .../import_mod/threatanalyzer_import.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index da01b93..326de89 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -15,7 +15,7 @@ misperrors = {'error': 'Error'} userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.6', 'author': 'Christophe Vandeplas', +moduleinfo = {'version': '0.7', 'author': 'Christophe Vandeplas', 'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files', 'module-type': ['import']} @@ -451,23 +451,24 @@ def cleanup_filepath(item): def cleanup_regkey(item): noise_substrings = { - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', + r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith', r'\\CurrentVersion\\Explorer\\RecentDocs\\', r'\\CurrentVersion\\Explorer\\UserAssist\\', - r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith', - r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', - r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', - r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\', - r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', - r'\\Usage\\SpellingAndGrammarFiles', - r'^HKLM\\Software\\Microsoft\\Tracing\\', + r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', r'\\Software\\Classes\\CLSID\\', r'\\Software\\Classes\\Local Settings\\MuiCache\\', - r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', + r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', + r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', + r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', + r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\', + r'\\Software\\Microsoft\\Shared Tools\\Panose\\', + r'\\Software\\Microsoft\\Tracing\\', + r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\', r'\\Software\\Microsoft\\Tracing\\powershell_RASMANCS\\', - r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\' + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', + r'\\Usage\\SpellingAndGrammarFiles' } if list_in_string(noise_substrings, item, regex=True): return None From 67cecc89d0ea492cd7f77fd42b4f91a9814e339a Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 15 May 2018 12:59:55 +0200 Subject: [PATCH 4/5] threatanalyzer_import - minor generic noise removal --- .../import_mod/threatanalyzer_import.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index da01b93..326de89 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -15,7 +15,7 @@ misperrors = {'error': 'Error'} userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.6', 'author': 'Christophe Vandeplas', +moduleinfo = {'version': '0.7', 'author': 'Christophe Vandeplas', 'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files', 'module-type': ['import']} @@ -451,23 +451,24 @@ def cleanup_filepath(item): def cleanup_regkey(item): noise_substrings = { - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', + r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith', r'\\CurrentVersion\\Explorer\\RecentDocs\\', r'\\CurrentVersion\\Explorer\\UserAssist\\', - r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith', - r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', - r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', - r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\', - r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', - r'\\Usage\\SpellingAndGrammarFiles', - r'^HKLM\\Software\\Microsoft\\Tracing\\', + r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', r'\\Software\\Classes\\CLSID\\', r'\\Software\\Classes\\Local Settings\\MuiCache\\', - r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', + r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', + r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', + r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', + r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\', + r'\\Software\\Microsoft\\Shared Tools\\Panose\\', + r'\\Software\\Microsoft\\Tracing\\', + r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\', r'\\Software\\Microsoft\\Tracing\\powershell_RASMANCS\\', - r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\' + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', + r'\\Usage\\SpellingAndGrammarFiles' } if list_in_string(noise_substrings, item, regex=True): return None From 0593dbb40807b01988b88902e8732e150538e6e2 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 16 May 2018 11:50:47 +0200 Subject: [PATCH 5/5] ta import - more filter for pollution --- .../import_mod/threatanalyzer_import.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 326de89..757f849 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -421,20 +421,22 @@ def cleanup_url(item): def cleanup_filepath(item): noise_substrings = { - 'C:\\Windows\\Prefetch\\', - '\\AppData\\Roaming\\Microsoft\\Windows\\Recent\\', - '\\AppData\\Roaming\\Microsoft\\Office\\Recent\\', - 'C:\\ProgramData\\Microsoft\\OfficeSoftwareProtectionPlatform\\Cache\\cache.dat', - '\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content.', - '\\AppData\\Local\\Microsoft\\Internet Explorer\\Recovery\\High\\', + '\\AppData\\Local\\GDIPFONTCACHEV1.DAT', '\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\', - '\\AppData\\LocalLow\\Microsoft\\Internet Explorer\\Services\\search_', - '\\AppData\\Local\\Microsoft\\Windows\\History\\History.', - '\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\', - '\\AppData\\LocalLow\\Microsoft\\CryptnetUrlCache\\', + '\\AppData\\Local\\Microsoft\\Internet Explorer\\Recovery\\High\\', '\\AppData\\Local\\Microsoft\\Windows\\Caches\\', - '\\AppData\\Local\\Microsoft\\Windows\WebCache\\', '\\AppData\\Local\\Microsoft\\Windows\\Explorer\\thumbcache', + '\\AppData\\Local\\Microsoft\\Windows\\History\\History.', + '\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content.', + '\\AppData\\Local\\Microsoft\\Windows\\WebCache\\', + '\\AppData\\Local\\Temp\\.*tmp$', + '\\AppData\\LocalLow\\Microsoft\\CryptnetUrlCache\\', + '\\AppData\\LocalLow\\Microsoft\\Internet Explorer\\Services\\search_', + '\\AppData\\Roaming\\Microsoft\\Office\\Recent\\', + '\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\', + '\\AppData\\Roaming\\Microsoft\\Windows\\Recent\\', + 'C:\\ProgramData\\Microsoft\\OfficeSoftwareProtectionPlatform\\Cache\\cache.dat', + 'C:\\Windows\\Prefetch\\', '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\SharedDataEvents-journal', '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin', @@ -460,14 +462,16 @@ def cleanup_regkey(item): r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', - r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\', + r'\\Software\\Microsoft\\OfficeSoftwareProtectionPlatform\\', r'\\Software\\Microsoft\\Shared Tools\\Panose\\', r'\\Software\\Microsoft\\Tracing\\', r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\', r'\\Software\\Microsoft\\Tracing\\powershell_RASMANCS\\', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Action Center\\', r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', + r'\\System\\CurrentControlSet\\Services\\RdyBoost\\', r'\\Usage\\SpellingAndGrammarFiles' } if list_in_string(noise_substrings, item, regex=True):