Merge pull request #628 from righel/add-sigmf-expand-module

new: add sigmf module to expand a sigmf recording object template
pull/634/head
Luciano Righetti 2023-08-03 09:37:50 +02:00 committed by GitHub
commit 10c333cd1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 299 additions and 1 deletions

View File

@ -71,6 +71,7 @@ maclookup==1.0.3
markdown-it-py==2.2.0 ; python_version >= '3.7' markdown-it-py==2.2.0 ; python_version >= '3.7'
markdownify==0.5.3 markdownify==0.5.3
markupsafe==2.1.2 ; python_version >= '3.7' markupsafe==2.1.2 ; python_version >= '3.7'
matplotlib==3.7.2
mattermostdriver==7.3.2 mattermostdriver==7.3.2
maxminddb==2.3.0 ; python_version >= '3.7' maxminddb==2.3.0 ; python_version >= '3.7'
mdurl==0.1.2 ; python_version >= '3.7' mdurl==0.1.2 ; python_version >= '3.7'
@ -147,6 +148,7 @@ secretstorage==3.3.3 ; sys_platform == 'linux'
setuptools==67.7.2 ; python_version >= '3.7' setuptools==67.7.2 ; python_version >= '3.7'
shodan==1.29.1 shodan==1.29.1
sigmatools==0.19.1 sigmatools==0.19.1
sigmf==1.1.1
simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3' simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sniffio==1.3.0 ; python_version >= '3.7' sniffio==1.3.0 ; python_version >= '3.7'

View File

@ -0,0 +1,294 @@
# -*- coding: utf-8 -*-
import base64
import numpy as np
import matplotlib.pyplot as plt
import io
import json
import tempfile
import logging
import sys
from pymisp import MISPObject, MISPEvent
from sigmf import SigMFFile
from sigmf.archive import SIGMF_DATASET_EXT, SIGMF_METADATA_EXT
import tarfile
import codecs
log = logging.getLogger("sigmf-expand")
log.setLevel(logging.DEBUG)
sh = logging.StreamHandler(sys.stdout)
sh.setLevel(logging.DEBUG)
fmt = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
sh.setFormatter(fmt)
log.addHandler(sh)
misperrors = {'error': 'Error'}
mispattributes = {'input': ['sigmf-recording', 'sigmf-archive'], 'output': [
'MISP objects'], 'format': 'misp_standard'}
moduleinfo = {'version': '0.1', 'author': 'Luciano Righetti',
'description': 'Expands a SigMF Recording object into a SigMF Expanded Recording object, extracts a SigMF archive into a SigMF Recording object.',
'module-type': ['expansion']}
def get_samples(data_bytes, data_type) -> np.ndarray:
"""
Get samples from bytes.
Source: https://github.com/IQEngine/IQEngine/blob/main/api/rf/samples.py
Parameters
----------
data_bytes : bytes
The bytes to convert to samples.
data_type : str
The data type of the bytes.
Returns
-------
np.ndarray
The samples.
"""
if data_type == "ci16_le" or data_type == "ci16":
samples = np.frombuffer(data_bytes, dtype=np.int16)
samples = samples[::2] + 1j * samples[1::2]
elif data_type == "cf32_le":
samples = np.frombuffer(data_bytes, dtype=np.complex64)
elif data_type == "ci8" or data_type == "i8":
samples = np.frombuffer(data_bytes, dtype=np.int8)
samples = samples[::2] + 1j * samples[1::2]
else:
raise ("Datatype " + data_type + " not implemented")
return samples
def generate_plots(recording, meta_filename, data_bytes):
# FFT plot
filename = meta_filename.replace('.sigmf-data', '')
samples = get_samples(
data_bytes, recording.get_global_info()['core:datatype'])
sample_rate = recording.get_global_info()['core:sample_rate']
# Waterfall plot
# snippet from https://pysdr.org/content/frequency_domain.html#fast-fourier-transform-fft
fft_size = 1024
# // is an integer division which rounds down
num_rows = len(samples) // fft_size
spectrogram = np.zeros((num_rows, fft_size))
for i in range(num_rows):
spectrogram[i, :] = 10 * \
np.log10(np.abs(np.fft.fftshift(
np.fft.fft(samples[i*fft_size:(i+1)*fft_size])))**2)
plt.figure(figsize=(10, 4))
plt.title(filename)
plt.imshow(spectrogram, aspect='auto', extent=[
sample_rate/-2/1e6, sample_rate/2/1e6, 0, len(samples)/sample_rate])
plt.xlabel("Frequency [MHz]")
plt.ylabel("Time [ms]")
plt.savefig(filename + '-spectrogram.png')
waterfall_buff = io.BytesIO()
plt.savefig(waterfall_buff, format='png')
waterfall_buff.seek(0)
waterfall_png = base64.b64encode(waterfall_buff.read()).decode('utf-8')
waterfall_attr = {
'type': 'attachment',
'value': filename + '-waterfall.png',
'data': waterfall_png,
'comment': 'Waterfall plot of the recording'
}
return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}]
def process_sigmf_archive(object):
event = MISPEvent()
sigmf_data_attr = None
sigmf_meta_attr = None
try:
# get sigmf-archive attribute
for attribute in object['Attribute']:
if attribute['object_relation'] == 'SigMF-archive':
# write temp data file to disk
sigmf_archive_file = tempfile.NamedTemporaryFile(
suffix='.sigmf')
sigmf_archive_bin = base64.b64decode(attribute['data'])
with open(sigmf_archive_file.name, 'wb') as f:
f.write(sigmf_archive_bin)
f.close()
sigmf_tarfile = tarfile.open(
sigmf_archive_file.name, mode="r", format=tarfile.PAX_FORMAT)
files = sigmf_tarfile.getmembers()
for file in files:
if file.name.endswith(SIGMF_METADATA_EXT):
metadata_reader = sigmf_tarfile.extractfile(file)
sigmf_meta_attr = {
'type': 'attachment',
'value': file.name,
'data': base64.b64encode(metadata_reader.read()).decode("utf-8"),
'comment': 'SigMF metadata file',
'object_relation': 'SigMF-meta'
}
if file.name.endswith(SIGMF_DATASET_EXT):
data_reader = sigmf_tarfile.extractfile(file)
sigmf_data_attr = {
'type': 'attachment',
'value': file.name,
'data': base64.b64encode(data_reader.read()).decode("utf-8"),
'comment': 'SigMF data file',
'object_relation': 'SigMF-data'
}
if sigmf_meta_attr is None:
return {"error": "No SigMF metadata file found"}
recording = MISPObject('sigmf-recording')
recording.add_attribute(**sigmf_meta_attr)
recording.add_attribute(**sigmf_data_attr)
# add reference to original SigMF Archive object
recording.add_reference(object['uuid'], "expands")
event.add_object(recording)
event = json.loads(event.to_json())
return {"results": {'Object': event['Object']}}
# no sigmf-archive attribute found
return {"error": "No SigMF-archive attribute found"}
except Exception as e:
logging.exception(e)
return {"error": "An error occured when processing the SigMF archive"}
def process_sigmf_recording(object):
event = MISPEvent()
for attribute in object['Attribute']:
if attribute['object_relation'] == 'SigMF-data':
sigmf_data_attr = attribute
if attribute['object_relation'] == 'SigMF-meta':
sigmf_meta_attr = attribute
if sigmf_meta_attr is None:
return {"error": "No SigMF-data attribute"}
if sigmf_data_attr is None:
return {"error": "No SigMF-meta attribute"}
try:
sigmf_meta = base64.b64decode(sigmf_meta_attr['data']).decode('utf-8')
sigmf_meta = json.loads(sigmf_meta)
except Exception as e:
logging.exception(e)
return {"error": "Provided .sigmf-meta is not a valid JSON string"}
# write temp data file to disk
sigmf_data_file = tempfile.NamedTemporaryFile(suffix='.sigmf-data')
sigmf_data_bin = base64.b64decode(sigmf_data_attr['data'])
with open(sigmf_data_file.name, 'wb') as f:
f.write(sigmf_data_bin)
f.close()
try:
recording = SigMFFile(
metadata=sigmf_meta,
data_file=sigmf_data_file.name
)
except Exception as e:
logging.exception(e)
return {"error": "Provided .sigmf-meta and .sigmf-data is not a valid SigMF file"}
expanded_sigmf = MISPObject('sigmf-expanded-recording')
if 'core:author' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'author', **{'type': 'text', 'value': sigmf_meta['global']['core:author']})
if 'core:datatype' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'datatype', **{'type': 'text', 'value': sigmf_meta['global']['core:datatype']})
if 'core:description' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'description', **{'type': 'text', 'value': sigmf_meta['global']['core:description']})
if 'core:license' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'license', **{'type': 'text', 'value': sigmf_meta['global']['core:license']})
if 'core:num_channels' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'num_channels', **{'type': 'counter', 'value': sigmf_meta['global']['core:num_channels']})
if 'core:recorder' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'recorder', **{'type': 'text', 'value': sigmf_meta['global']['core:recorder']})
if 'core:sample_rate' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'sample_rate', **{'type': 'float', 'value': sigmf_meta['global']['core:sample_rate']})
if 'core:sha512' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'sha512', **{'type': 'text', 'value': sigmf_meta['global']['core:sha512']})
if 'core:version' in sigmf_meta['global']:
expanded_sigmf.add_attribute(
'version', **{'type': 'text', 'value': sigmf_meta['global']['core:version']})
# add reference to original SigMF Recording object
expanded_sigmf.add_reference(object['uuid'], "expands")
# add FFT and waterfall plot
try:
plots = generate_plots(
recording, sigmf_data_attr['value'], sigmf_data_bin)
except Exception as e:
logging.exception(e)
return {"error": "Could not generate plots"}
for plot in plots:
expanded_sigmf.add_attribute(plot['relation'], **plot['attribute'])
event.add_object(expanded_sigmf)
event = json.loads(event.to_json())
return {"results": {'Object': event['Object']}}
def handler(q=False):
request = json.loads(q)
object = request.get("object")
event = MISPEvent()
if not object:
return {"error": "No object provided"}
if 'Attribute' not in object:
return {"error": "Empty Attribute list"}
# check if it's a SigMF Archive
if object['name'] == 'sigmf-archive':
return process_sigmf_archive(object)
# check if it's a SigMF Recording
if object['name'] == 'sigmf-recording':
return process_sigmf_recording(object)
# TODO: add support for SigMF Collection
return {"error": "No SigMF object provided"}
def introspection():
return mispattributes
def version():
return moduleinfo

View File

@ -578,13 +578,14 @@ class TestExpansions(unittest.TestCase):
query_values = ('www.bestwpdesign.com', '79.118.195.239', query_values = ('www.bestwpdesign.com', '79.118.195.239',
'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
'http://79.118.195.239:1924/.i') 'http://79.118.195.239:1924/.i')
results = ('url', 'url', 'virustotal-report', 'virustotal-report') results = ('url', 'url', 'file', 'virustotal-report')
for query_type, query_value, result in zip(query_types[:2], query_values[:2], results[:2]): for query_type, query_value, result in zip(query_types[:2], query_values[:2], results[:2]):
query = {"module": "urlhaus", query = {"module": "urlhaus",
"attribute": {"type": query_type, "attribute": {"type": query_type,
"value": query_value, "value": query_value,
"uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
print(response.json())
self.assertEqual(self.get_attribute(response), result) self.assertEqual(self.get_attribute(response), result)
for query_type, query_value, result in zip(query_types[2:], query_values[2:], results[2:]): for query_type, query_value, result in zip(query_types[2:], query_values[2:], results[2:]):
query = {"module": "urlhaus", query = {"module": "urlhaus",
@ -592,6 +593,7 @@ class TestExpansions(unittest.TestCase):
"value": query_value, "value": query_value,
"uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
print(response.json())
self.assertEqual(self.get_object(response), result) self.assertEqual(self.get_object(response), result)
def test_urlscan(self): def test_urlscan(self):