Merge pull request #728 from marjatech/main

update Malpedia incl automation

tools/del_duplicate_refs.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# coding=utf-8
+"""
+    Tool to remove duplicates in cluster references
+"""
+import sys
+import json
+
+with open(sys.argv[1], 'r') as f:
+    data = json.load(f)
+
+for c in data['values']:
+    c['meta']['refs'] = list(dict.fromkeys(c['meta']['refs']))
+
+with open(sys.argv[1], 'w') as f:
+    json.dump(data, f)
+

tools/fetch_malpedia.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd "${0%/*}"
+wget -O malpedia.json https://malpedia.caad.fkie.fraunhofer.de/api/get/misp
+mv malpedia.json ../clusters/malpedia.json
+./del_duplicate_refs.py ../clusters/malpedia.json
+(cd ..; ./jq_all_the_things.sh)