From 5e2957b13f1a4ad63b7dc096688568fbf8997402 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Tue, 11 Jul 2023 16:42:33 +0200 Subject: [PATCH 1/7] new: add sigmf module to expand a sigmf recording object template --- .../modules/expansion/sigmf-expand.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 misp_modules/modules/expansion/sigmf-expand.py diff --git a/misp_modules/modules/expansion/sigmf-expand.py b/misp_modules/modules/expansion/sigmf-expand.py new file mode 100644 index 0000000..709e6cc --- /dev/null +++ b/misp_modules/modules/expansion/sigmf-expand.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +import base64 +import json +import tempfile +import logging +import sys +from pymisp import MISPObject, MISPEvent +from sigmf import SigMFFile + +log = logging.getLogger("sigmf-expand") +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +sh.setFormatter(fmt) +log.addHandler(sh) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['sigmf-recording'], 'output': [ + 'MISP objects'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Luciano Righetti', + 'description': 'Expand a SigMF Recording object into a SigMF Expanded Recording object.', + 'module-type': ['expansion']} + + +def handler(q=False): + request = json.loads(q) + object = request.get("object") + if not object: + return {"error": "No object provided"} + + if 'Attribute' not in object: + return {"error": "Empty Attribute list"} + + for attribute in object['Attribute']: + if attribute['object_relation'] == 'SigMF-data': + sigmf_data_attr = attribute + + if attribute['object_relation'] == 'SigMF-meta': + sigmf_meta_attr = attribute + + if sigmf_meta_attr is None: + return {"error": "No SigMF-data attribute"} + + if sigmf_data_attr is None: + return {"error": "No SigMF-meta attribute"} + + try: + sigmf_meta = base64.b64decode(sigmf_meta_attr['data']).decode('utf-8') + sigmf_meta = json.loads(sigmf_meta) + except Exception as e: + logging.exception(e) + return {"error": "Provided .sigmf-meta is not a valid JSON string"} + + # write temp data file to disk + sigmf_data_file = tempfile.NamedTemporaryFile(suffix='.sigmf-data') + sigmf_data_bin = base64.b64decode(sigmf_data_attr['data']) + with open(sigmf_data_file.name, 'wb') as f: + f.write(sigmf_data_bin) + f.close() + + try: + recording = SigMFFile( + metadata=sigmf_meta, + data_file=sigmf_data_file.name + ) + except Exception as e: + logging.exception(e) + return {"error": "Provided .sigmf-meta and .sigmf-data is not a valid SigMF file"} + + event = MISPEvent() + expanded_sigmf = MISPObject('sigmf-expanded-recording') + + if 'core:author' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'author', **{'type': 'text', 'value': sigmf_meta['global']['core:author']}) + if 'core:datatype' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'datatype', **{'type': 'text', 'value': sigmf_meta['global']['core:datatype']}) + if 'core:description' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'description', **{'type': 'text', 'value': sigmf_meta['global']['core:description']}) + if 'core:license' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'license', **{'type': 'text', 'value': sigmf_meta['global']['core:license']}) + if 'core:num_channels' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'num_channels', **{'type': 'counter', 'value': sigmf_meta['global']['core:num_channels']}) + if 'core:recorder' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'recorder', **{'type': 'text', 'value': sigmf_meta['global']['core:recorder']}) + if 'core:sample_rate' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'sample_rate', **{'type': 'float', 'value': sigmf_meta['global']['core:sample_rate']}) + if 'core:sha512' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'sha512', **{'type': 'text', 'value': sigmf_meta['global']['core:sha512']}) + if 'core:version' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'version', **{'type': 'text', 'value': sigmf_meta['global']['core:version']}) + + # TODO: geolocation (GeoJSON) + + # add reference to original SigMF Recording object + expanded_sigmf.add_reference(object['uuid'], "expands") + + event.add_object(expanded_sigmf) + event = json.loads(event.to_json()) + + return {"results": {'Object': event['Object']}} + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo From 3f0fa1454582f3fff96e9f611e37f0b225400b3c Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 12 Jul 2023 15:34:44 +0200 Subject: [PATCH 2/7] new: add waterfall plot to the expanded object --- .../modules/expansion/sigmf-expand.py | 85 ++++++++++++++++++- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/sigmf-expand.py b/misp_modules/modules/expansion/sigmf-expand.py index 709e6cc..8efbf58 100644 --- a/misp_modules/modules/expansion/sigmf-expand.py +++ b/misp_modules/modules/expansion/sigmf-expand.py @@ -1,12 +1,16 @@ # -*- coding: utf-8 -*- import base64 +import numpy as np +import matplotlib.pyplot as plt +import io import json import tempfile import logging import sys from pymisp import MISPObject, MISPEvent from sigmf import SigMFFile +import pymisp log = logging.getLogger("sigmf-expand") log.setLevel(logging.DEBUG) @@ -26,6 +30,71 @@ moduleinfo = {'version': '0.1', 'author': 'Luciano Righetti', 'module-type': ['expansion']} +def generate_plots(recording, meta_filename): + # FFT plot + filename = meta_filename.replace('.sigmf-data', '') + # snippet from https://gist.github.com/daniestevez/0d519fd4044f3b9f44e170fd619fbb40 + NFFT = 2048 + N = NFFT * 4096 + fs = recording.get_global_info()['core:sample_rate'] + x = np.fromfile(recording.data_file, 'int16', count=2*N) + x = x[::2] + 1j * x[1::2] + + # f = np.fft.fftshift(np.average( + # np.abs(np.fft.fft(x.reshape(-1, NFFT)))**2, axis=0)) + # freq = np.fft.fftshift(np.fft.fftfreq(NFFT, 1/fs)) + + # plt.figure(figsize=(10, 4)) + # plt.plot(1e-6 * freq, 10*np.log10(f)) + # plt.title(filename) + # plt.ylabel('PSD (dB)') + # plt.xlabel('Baseband frequency (MHz)') + # fft_buff = io.BytesIO() + # plt.savefig(fft_buff, format='png') + # fft_buff.seek(0) + # fft_png = base64.b64encode(fft_buff.read()).decode('utf-8') + + # fft_attr = { + # 'type': 'attachment', + # 'value': filename + '-fft.png', + # 'data': fft_png, + # 'comment': 'FFT plot of the recording' + # } + + # Waterfall plot + # snippet from https://pysdr.org/content/frequency_domain.html#fast-fourier-transform-fft + fft_size = 1024 + # // is an integer division which rounds down + num_rows = len(x) // fft_size + spectrogram = np.zeros((num_rows, fft_size)) + for i in range(num_rows): + spectrogram[i, :] = 10 * \ + np.log10(np.abs(np.fft.fftshift( + np.fft.fft(x[i*fft_size:(i+1)*fft_size])))**2) + + plt.figure(figsize=(10, 4)) + plt.title(filename) + plt.imshow(spectrogram, aspect='auto', extent=[ + fs/-2/1e6, fs/2/1e6, 0, len(x)/fs]) + plt.xlabel("Frequency [MHz]") + plt.ylabel("Time [ms]") + plt.savefig(filename + '-spectrogram.png') + waterfall_buff = io.BytesIO() + plt.savefig(waterfall_buff, format='png') + waterfall_buff.seek(0) + waterfall_png = base64.b64encode(waterfall_buff.read()).decode('utf-8') + + waterfall_attr = { + 'type': 'attachment', + 'value': filename + '-waterfall.png', + 'data': waterfall_png, + 'comment': 'Waterfall plot of the recording' + } + + # return [fft_attr, waterfall_attr] + return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}] + + def handler(q=False): request = json.loads(q) object = request.get("object") @@ -73,6 +142,8 @@ def handler(q=False): event = MISPEvent() expanded_sigmf = MISPObject('sigmf-expanded-recording') + logging.error(expanded_sigmf.to_json()) + logging.error(pymisp.__file__) if 'core:author' in sigmf_meta['global']: expanded_sigmf.add_attribute( @@ -102,11 +173,19 @@ def handler(q=False): expanded_sigmf.add_attribute( 'version', **{'type': 'text', 'value': sigmf_meta['global']['core:version']}) - # TODO: geolocation (GeoJSON) - # add reference to original SigMF Recording object expanded_sigmf.add_reference(object['uuid'], "expands") - + + # add FFT and waterfall plot + try: + plots = generate_plots(recording, sigmf_data_attr['value']) + except Exception as e: + logging.exception(e) + return {"error": "Could not generate plots"} + + for plot in plots: + expanded_sigmf.add_attribute(plot['relation'], **plot['attribute']) + event.add_object(expanded_sigmf) event = json.loads(event.to_json()) From e26bfef477a83011e66ada43734e24b6cf3944a1 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 12 Jul 2023 15:51:50 +0200 Subject: [PATCH 3/7] fix: remove debug --- misp_modules/modules/expansion/sigmf-expand.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misp_modules/modules/expansion/sigmf-expand.py b/misp_modules/modules/expansion/sigmf-expand.py index 8efbf58..def1e1b 100644 --- a/misp_modules/modules/expansion/sigmf-expand.py +++ b/misp_modules/modules/expansion/sigmf-expand.py @@ -142,8 +142,6 @@ def handler(q=False): event = MISPEvent() expanded_sigmf = MISPObject('sigmf-expanded-recording') - logging.error(expanded_sigmf.to_json()) - logging.error(pymisp.__file__) if 'core:author' in sigmf_meta['global']: expanded_sigmf.add_attribute( From df2183ce5488a7dc8d42e34dd6f2609a1ac3b5d2 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 13 Jul 2023 11:06:25 +0200 Subject: [PATCH 4/7] fix: properly read samples in different datatypes --- .../modules/expansion/sigmf-expand.py | 74 ++++++++++--------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/misp_modules/modules/expansion/sigmf-expand.py b/misp_modules/modules/expansion/sigmf-expand.py index def1e1b..ec51106 100644 --- a/misp_modules/modules/expansion/sigmf-expand.py +++ b/misp_modules/modules/expansion/sigmf-expand.py @@ -30,52 +30,60 @@ moduleinfo = {'version': '0.1', 'author': 'Luciano Righetti', 'module-type': ['expansion']} -def generate_plots(recording, meta_filename): +def get_samples(data_bytes, data_type) -> np.ndarray: + """ + Get samples from bytes. + + Source: https://github.com/IQEngine/IQEngine/blob/main/api/rf/samples.py + + Parameters + ---------- + data_bytes : bytes + The bytes to convert to samples. + data_type : str + The data type of the bytes. + + Returns + ------- + np.ndarray + The samples. + """ + + if data_type == "ci16_le" or data_type == "ci16": + samples = np.frombuffer(data_bytes, dtype=np.int16) + samples = samples[::2] + 1j * samples[1::2] + elif data_type == "cf32_le": + samples = np.frombuffer(data_bytes, dtype=np.complex64) + elif data_type == "ci8" or data_type == "i8": + samples = np.frombuffer(data_bytes, dtype=np.int8) + samples = samples[::2] + 1j * samples[1::2] + else: + raise ("Datatype " + data_type + " not implemented") + return samples + + +def generate_plots(recording, meta_filename, data_bytes): # FFT plot filename = meta_filename.replace('.sigmf-data', '') - # snippet from https://gist.github.com/daniestevez/0d519fd4044f3b9f44e170fd619fbb40 - NFFT = 2048 - N = NFFT * 4096 - fs = recording.get_global_info()['core:sample_rate'] - x = np.fromfile(recording.data_file, 'int16', count=2*N) - x = x[::2] + 1j * x[1::2] - - # f = np.fft.fftshift(np.average( - # np.abs(np.fft.fft(x.reshape(-1, NFFT)))**2, axis=0)) - # freq = np.fft.fftshift(np.fft.fftfreq(NFFT, 1/fs)) - - # plt.figure(figsize=(10, 4)) - # plt.plot(1e-6 * freq, 10*np.log10(f)) - # plt.title(filename) - # plt.ylabel('PSD (dB)') - # plt.xlabel('Baseband frequency (MHz)') - # fft_buff = io.BytesIO() - # plt.savefig(fft_buff, format='png') - # fft_buff.seek(0) - # fft_png = base64.b64encode(fft_buff.read()).decode('utf-8') - - # fft_attr = { - # 'type': 'attachment', - # 'value': filename + '-fft.png', - # 'data': fft_png, - # 'comment': 'FFT plot of the recording' - # } + samples = get_samples( + data_bytes, recording.get_global_info()['core:datatype']) + sample_rate = recording.get_global_info()['core:sample_rate'] # Waterfall plot # snippet from https://pysdr.org/content/frequency_domain.html#fast-fourier-transform-fft fft_size = 1024 # // is an integer division which rounds down - num_rows = len(x) // fft_size + num_rows = len(samples) // fft_size spectrogram = np.zeros((num_rows, fft_size)) for i in range(num_rows): spectrogram[i, :] = 10 * \ np.log10(np.abs(np.fft.fftshift( - np.fft.fft(x[i*fft_size:(i+1)*fft_size])))**2) + np.fft.fft(samples[i*fft_size:(i+1)*fft_size])))**2) plt.figure(figsize=(10, 4)) plt.title(filename) plt.imshow(spectrogram, aspect='auto', extent=[ - fs/-2/1e6, fs/2/1e6, 0, len(x)/fs]) + sample_rate/-2/1e6, sample_rate/2/1e6, 0, len(samples)/sample_rate]) plt.xlabel("Frequency [MHz]") plt.ylabel("Time [ms]") plt.savefig(filename + '-spectrogram.png') @@ -91,7 +99,6 @@ def generate_plots(recording, meta_filename): 'comment': 'Waterfall plot of the recording' } - # return [fft_attr, waterfall_attr] return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}] @@ -176,7 +183,8 @@ def handler(q=False): # add FFT and waterfall plot try: - plots = generate_plots(recording, sigmf_data_attr['value']) + plots = generate_plots( + recording, sigmf_data_attr['value'], sigmf_data_bin) except Exception as e: logging.exception(e) return {"error": "Could not generate plots"} From 6d9c64f6d6fa766fc826d061343c411ca3acc859 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Tue, 1 Aug 2023 14:35:56 +0200 Subject: [PATCH 5/7] add: add required python packages for sigmf expansion module --- REQUIREMENTS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 620d7a6..0084ffe 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -71,6 +71,7 @@ maclookup==1.0.3 markdown-it-py==2.2.0 ; python_version >= '3.7' markdownify==0.5.3 markupsafe==2.1.2 ; python_version >= '3.7' +matplotlib==3.7.2 mattermostdriver==7.3.2 maxminddb==2.3.0 ; python_version >= '3.7' mdurl==0.1.2 ; python_version >= '3.7' @@ -146,6 +147,7 @@ secretstorage==3.3.3 ; sys_platform == 'linux' setuptools==67.7.2 ; python_version >= '3.7' shodan==1.29.1 sigmatools==0.19.1 +sigmf==1.1.1 simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3' six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' sniffio==1.3.0 ; python_version >= '3.7' From 858b4ed1c68cb9e6eaf460912d3ba691eb27ac8b Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Tue, 1 Aug 2023 16:19:43 +0200 Subject: [PATCH 6/7] fix: ci, urlhaus api response changed --- tests/test_expansions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 5f4d326..0745361 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -577,13 +577,14 @@ class TestExpansions(unittest.TestCase): query_values = ('www.bestwpdesign.com', '79.118.195.239', 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', 'http://79.118.195.239:1924/.i') - results = ('url', 'url', 'virustotal-report', 'virustotal-report') + results = ('url', 'url', 'file', 'virustotal-report') for query_type, query_value, result in zip(query_types[:2], query_values[:2], results[:2]): query = {"module": "urlhaus", "attribute": {"type": query_type, "value": query_value, "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} response = self.misp_modules_post(query) + print(response.json()) self.assertEqual(self.get_attribute(response), result) for query_type, query_value, result in zip(query_types[2:], query_values[2:], results[2:]): query = {"module": "urlhaus", @@ -591,6 +592,7 @@ class TestExpansions(unittest.TestCase): "value": query_value, "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} response = self.misp_modules_post(query) + print(response.json()) self.assertEqual(self.get_object(response), result) def test_urlscan(self): From 23069a7c5d405e3228c733d38fce97f284a75f95 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 3 Aug 2023 09:25:46 +0200 Subject: [PATCH 7/7] add: support extracting sigmf archives into sigmf recordings --- .../modules/expansion/sigmf-expand.py | 110 ++++++++++++++++-- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/misp_modules/modules/expansion/sigmf-expand.py b/misp_modules/modules/expansion/sigmf-expand.py index ec51106..366ec18 100644 --- a/misp_modules/modules/expansion/sigmf-expand.py +++ b/misp_modules/modules/expansion/sigmf-expand.py @@ -10,7 +10,9 @@ import logging import sys from pymisp import MISPObject, MISPEvent from sigmf import SigMFFile -import pymisp +from sigmf.archive import SIGMF_DATASET_EXT, SIGMF_METADATA_EXT +import tarfile +import codecs log = logging.getLogger("sigmf-expand") log.setLevel(logging.DEBUG) @@ -23,10 +25,10 @@ sh.setFormatter(fmt) log.addHandler(sh) misperrors = {'error': 'Error'} -mispattributes = {'input': ['sigmf-recording'], 'output': [ +mispattributes = {'input': ['sigmf-recording', 'sigmf-archive'], 'output': [ 'MISP objects'], 'format': 'misp_standard'} moduleinfo = {'version': '0.1', 'author': 'Luciano Righetti', - 'description': 'Expand a SigMF Recording object into a SigMF Expanded Recording object.', + 'description': 'Expands a SigMF Recording object into a SigMF Expanded Recording object, extracts a SigMF archive into a SigMF Recording object.', 'module-type': ['expansion']} @@ -102,14 +104,77 @@ def generate_plots(recording, meta_filename, data_bytes): return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}] -def handler(q=False): - request = json.loads(q) - object = request.get("object") - if not object: - return {"error": "No object provided"} +def process_sigmf_archive(object): - if 'Attribute' not in object: - return {"error": "Empty Attribute list"} + event = MISPEvent() + sigmf_data_attr = None + sigmf_meta_attr = None + + try: + # get sigmf-archive attribute + for attribute in object['Attribute']: + if attribute['object_relation'] == 'SigMF-archive': + + # write temp data file to disk + sigmf_archive_file = tempfile.NamedTemporaryFile( + suffix='.sigmf') + sigmf_archive_bin = base64.b64decode(attribute['data']) + with open(sigmf_archive_file.name, 'wb') as f: + f.write(sigmf_archive_bin) + f.close() + + sigmf_tarfile = tarfile.open( + sigmf_archive_file.name, mode="r", format=tarfile.PAX_FORMAT) + + files = sigmf_tarfile.getmembers() + + for file in files: + if file.name.endswith(SIGMF_METADATA_EXT): + metadata_reader = sigmf_tarfile.extractfile(file) + sigmf_meta_attr = { + 'type': 'attachment', + 'value': file.name, + 'data': base64.b64encode(metadata_reader.read()).decode("utf-8"), + 'comment': 'SigMF metadata file', + 'object_relation': 'SigMF-meta' + } + + if file.name.endswith(SIGMF_DATASET_EXT): + data_reader = sigmf_tarfile.extractfile(file) + sigmf_data_attr = { + 'type': 'attachment', + 'value': file.name, + 'data': base64.b64encode(data_reader.read()).decode("utf-8"), + 'comment': 'SigMF data file', + 'object_relation': 'SigMF-data' + } + + if sigmf_meta_attr is None: + return {"error": "No SigMF metadata file found"} + + recording = MISPObject('sigmf-recording') + recording.add_attribute(**sigmf_meta_attr) + recording.add_attribute(**sigmf_data_attr) + + # add reference to original SigMF Archive object + recording.add_reference(object['uuid'], "expands") + + event.add_object(recording) + event = json.loads(event.to_json()) + + return {"results": {'Object': event['Object']}} + + # no sigmf-archive attribute found + return {"error": "No SigMF-archive attribute found"} + + except Exception as e: + logging.exception(e) + return {"error": "An error occured when processing the SigMF archive"} + + +def process_sigmf_recording(object): + + event = MISPEvent() for attribute in object['Attribute']: if attribute['object_relation'] == 'SigMF-data': @@ -147,7 +212,6 @@ def handler(q=False): logging.exception(e) return {"error": "Provided .sigmf-meta and .sigmf-data is not a valid SigMF file"} - event = MISPEvent() expanded_sigmf = MISPObject('sigmf-expanded-recording') if 'core:author' in sigmf_meta['global']: @@ -198,6 +262,30 @@ def handler(q=False): return {"results": {'Object': event['Object']}} +def handler(q=False): + request = json.loads(q) + object = request.get("object") + event = MISPEvent() + + if not object: + return {"error": "No object provided"} + + if 'Attribute' not in object: + return {"error": "Empty Attribute list"} + + # check if it's a SigMF Archive + if object['name'] == 'sigmf-archive': + return process_sigmf_archive(object) + + # check if it's a SigMF Recording + if object['name'] == 'sigmf-recording': + return process_sigmf_recording(object) + + # TODO: add support for SigMF Collection + + return {"error": "No SigMF object provided"} + + def introspection(): return mispattributes