diff --git a/REQUIREMENTS b/REQUIREMENTS index 140a461..cad1348 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -71,6 +71,7 @@ maclookup==1.0.3 markdown-it-py==2.2.0 ; python_version >= '3.7' markdownify==0.5.3 markupsafe==2.1.2 ; python_version >= '3.7' +matplotlib==3.7.2 mattermostdriver==7.3.2 maxminddb==2.3.0 ; python_version >= '3.7' mdurl==0.1.2 ; python_version >= '3.7' @@ -147,6 +148,7 @@ secretstorage==3.3.3 ; sys_platform == 'linux' setuptools==67.7.2 ; python_version >= '3.7' shodan==1.29.1 sigmatools==0.19.1 +sigmf==1.1.1 simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3' six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' sniffio==1.3.0 ; python_version >= '3.7' diff --git a/misp_modules/modules/expansion/sigmf-expand.py b/misp_modules/modules/expansion/sigmf-expand.py new file mode 100644 index 0000000..366ec18 --- /dev/null +++ b/misp_modules/modules/expansion/sigmf-expand.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- + +import base64 +import numpy as np +import matplotlib.pyplot as plt +import io +import json +import tempfile +import logging +import sys +from pymisp import MISPObject, MISPEvent +from sigmf import SigMFFile +from sigmf.archive import SIGMF_DATASET_EXT, SIGMF_METADATA_EXT +import tarfile +import codecs + +log = logging.getLogger("sigmf-expand") +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +sh.setFormatter(fmt) +log.addHandler(sh) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['sigmf-recording', 'sigmf-archive'], 'output': [ + 'MISP objects'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Luciano Righetti', + 'description': 'Expands a SigMF Recording object into a SigMF Expanded Recording object, extracts a SigMF archive into a SigMF Recording object.', + 'module-type': ['expansion']} + + +def get_samples(data_bytes, data_type) -> np.ndarray: + """ + Get samples from bytes. + + Source: https://github.com/IQEngine/IQEngine/blob/main/api/rf/samples.py + + Parameters + ---------- + data_bytes : bytes + The bytes to convert to samples. + data_type : str + The data type of the bytes. + + Returns + ------- + np.ndarray + The samples. + """ + + if data_type == "ci16_le" or data_type == "ci16": + samples = np.frombuffer(data_bytes, dtype=np.int16) + samples = samples[::2] + 1j * samples[1::2] + elif data_type == "cf32_le": + samples = np.frombuffer(data_bytes, dtype=np.complex64) + elif data_type == "ci8" or data_type == "i8": + samples = np.frombuffer(data_bytes, dtype=np.int8) + samples = samples[::2] + 1j * samples[1::2] + else: + raise ("Datatype " + data_type + " not implemented") + return samples + + +def generate_plots(recording, meta_filename, data_bytes): + # FFT plot + filename = meta_filename.replace('.sigmf-data', '') + samples = get_samples( + data_bytes, recording.get_global_info()['core:datatype']) + sample_rate = recording.get_global_info()['core:sample_rate'] + + # Waterfall plot + # snippet from https://pysdr.org/content/frequency_domain.html#fast-fourier-transform-fft + fft_size = 1024 + # // is an integer division which rounds down + num_rows = len(samples) // fft_size + spectrogram = np.zeros((num_rows, fft_size)) + for i in range(num_rows): + spectrogram[i, :] = 10 * \ + np.log10(np.abs(np.fft.fftshift( + np.fft.fft(samples[i*fft_size:(i+1)*fft_size])))**2) + + plt.figure(figsize=(10, 4)) + plt.title(filename) + plt.imshow(spectrogram, aspect='auto', extent=[ + sample_rate/-2/1e6, sample_rate/2/1e6, 0, len(samples)/sample_rate]) + plt.xlabel("Frequency [MHz]") + plt.ylabel("Time [ms]") + plt.savefig(filename + '-spectrogram.png') + waterfall_buff = io.BytesIO() + plt.savefig(waterfall_buff, format='png') + waterfall_buff.seek(0) + waterfall_png = base64.b64encode(waterfall_buff.read()).decode('utf-8') + + waterfall_attr = { + 'type': 'attachment', + 'value': filename + '-waterfall.png', + 'data': waterfall_png, + 'comment': 'Waterfall plot of the recording' + } + + return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}] + + +def process_sigmf_archive(object): + + event = MISPEvent() + sigmf_data_attr = None + sigmf_meta_attr = None + + try: + # get sigmf-archive attribute + for attribute in object['Attribute']: + if attribute['object_relation'] == 'SigMF-archive': + + # write temp data file to disk + sigmf_archive_file = tempfile.NamedTemporaryFile( + suffix='.sigmf') + sigmf_archive_bin = base64.b64decode(attribute['data']) + with open(sigmf_archive_file.name, 'wb') as f: + f.write(sigmf_archive_bin) + f.close() + + sigmf_tarfile = tarfile.open( + sigmf_archive_file.name, mode="r", format=tarfile.PAX_FORMAT) + + files = sigmf_tarfile.getmembers() + + for file in files: + if file.name.endswith(SIGMF_METADATA_EXT): + metadata_reader = sigmf_tarfile.extractfile(file) + sigmf_meta_attr = { + 'type': 'attachment', + 'value': file.name, + 'data': base64.b64encode(metadata_reader.read()).decode("utf-8"), + 'comment': 'SigMF metadata file', + 'object_relation': 'SigMF-meta' + } + + if file.name.endswith(SIGMF_DATASET_EXT): + data_reader = sigmf_tarfile.extractfile(file) + sigmf_data_attr = { + 'type': 'attachment', + 'value': file.name, + 'data': base64.b64encode(data_reader.read()).decode("utf-8"), + 'comment': 'SigMF data file', + 'object_relation': 'SigMF-data' + } + + if sigmf_meta_attr is None: + return {"error": "No SigMF metadata file found"} + + recording = MISPObject('sigmf-recording') + recording.add_attribute(**sigmf_meta_attr) + recording.add_attribute(**sigmf_data_attr) + + # add reference to original SigMF Archive object + recording.add_reference(object['uuid'], "expands") + + event.add_object(recording) + event = json.loads(event.to_json()) + + return {"results": {'Object': event['Object']}} + + # no sigmf-archive attribute found + return {"error": "No SigMF-archive attribute found"} + + except Exception as e: + logging.exception(e) + return {"error": "An error occured when processing the SigMF archive"} + + +def process_sigmf_recording(object): + + event = MISPEvent() + + for attribute in object['Attribute']: + if attribute['object_relation'] == 'SigMF-data': + sigmf_data_attr = attribute + + if attribute['object_relation'] == 'SigMF-meta': + sigmf_meta_attr = attribute + + if sigmf_meta_attr is None: + return {"error": "No SigMF-data attribute"} + + if sigmf_data_attr is None: + return {"error": "No SigMF-meta attribute"} + + try: + sigmf_meta = base64.b64decode(sigmf_meta_attr['data']).decode('utf-8') + sigmf_meta = json.loads(sigmf_meta) + except Exception as e: + logging.exception(e) + return {"error": "Provided .sigmf-meta is not a valid JSON string"} + + # write temp data file to disk + sigmf_data_file = tempfile.NamedTemporaryFile(suffix='.sigmf-data') + sigmf_data_bin = base64.b64decode(sigmf_data_attr['data']) + with open(sigmf_data_file.name, 'wb') as f: + f.write(sigmf_data_bin) + f.close() + + try: + recording = SigMFFile( + metadata=sigmf_meta, + data_file=sigmf_data_file.name + ) + except Exception as e: + logging.exception(e) + return {"error": "Provided .sigmf-meta and .sigmf-data is not a valid SigMF file"} + + expanded_sigmf = MISPObject('sigmf-expanded-recording') + + if 'core:author' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'author', **{'type': 'text', 'value': sigmf_meta['global']['core:author']}) + if 'core:datatype' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'datatype', **{'type': 'text', 'value': sigmf_meta['global']['core:datatype']}) + if 'core:description' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'description', **{'type': 'text', 'value': sigmf_meta['global']['core:description']}) + if 'core:license' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'license', **{'type': 'text', 'value': sigmf_meta['global']['core:license']}) + if 'core:num_channels' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'num_channels', **{'type': 'counter', 'value': sigmf_meta['global']['core:num_channels']}) + if 'core:recorder' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'recorder', **{'type': 'text', 'value': sigmf_meta['global']['core:recorder']}) + if 'core:sample_rate' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'sample_rate', **{'type': 'float', 'value': sigmf_meta['global']['core:sample_rate']}) + if 'core:sha512' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'sha512', **{'type': 'text', 'value': sigmf_meta['global']['core:sha512']}) + if 'core:version' in sigmf_meta['global']: + expanded_sigmf.add_attribute( + 'version', **{'type': 'text', 'value': sigmf_meta['global']['core:version']}) + + # add reference to original SigMF Recording object + expanded_sigmf.add_reference(object['uuid'], "expands") + + # add FFT and waterfall plot + try: + plots = generate_plots( + recording, sigmf_data_attr['value'], sigmf_data_bin) + except Exception as e: + logging.exception(e) + return {"error": "Could not generate plots"} + + for plot in plots: + expanded_sigmf.add_attribute(plot['relation'], **plot['attribute']) + + event.add_object(expanded_sigmf) + event = json.loads(event.to_json()) + + return {"results": {'Object': event['Object']}} + + +def handler(q=False): + request = json.loads(q) + object = request.get("object") + event = MISPEvent() + + if not object: + return {"error": "No object provided"} + + if 'Attribute' not in object: + return {"error": "Empty Attribute list"} + + # check if it's a SigMF Archive + if object['name'] == 'sigmf-archive': + return process_sigmf_archive(object) + + # check if it's a SigMF Recording + if object['name'] == 'sigmf-recording': + return process_sigmf_recording(object) + + # TODO: add support for SigMF Collection + + return {"error": "No SigMF object provided"} + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 833bada..17563b6 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -578,13 +578,14 @@ class TestExpansions(unittest.TestCase): query_values = ('www.bestwpdesign.com', '79.118.195.239', 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', 'http://79.118.195.239:1924/.i') - results = ('url', 'url', 'virustotal-report', 'virustotal-report') + results = ('url', 'url', 'file', 'virustotal-report') for query_type, query_value, result in zip(query_types[:2], query_values[:2], results[:2]): query = {"module": "urlhaus", "attribute": {"type": query_type, "value": query_value, "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} response = self.misp_modules_post(query) + print(response.json()) self.assertEqual(self.get_attribute(response), result) for query_type, query_value, result in zip(query_types[2:], query_values[2:], results[2:]): query = {"module": "urlhaus", @@ -592,6 +593,7 @@ class TestExpansions(unittest.TestCase): "value": query_value, "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} response = self.misp_modules_post(query) + print(response.json()) self.assertEqual(self.get_object(response), result) def test_urlscan(self):