From c93103bba17c501a5cebe49b9646ccad1b8fe86e Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 30 Aug 2019 09:57:05 +0200 Subject: [PATCH] Add test for empty strings Should prevent MISP/misp-galaxy#438 --- .gitignore | 1 + tools/__init__.py | 0 tools/chk_dup.py | 33 ++++++++++++++++++++++++--------- tools/chk_empty_strings.py | 24 ++++++++++++++++++++++++ validate_all.sh | 3 +++ 5 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 .gitignore create mode 100644 tools/__init__.py create mode 100755 tools/chk_empty_strings.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/chk_dup.py b/tools/chk_dup.py index 2ed2f89..9df3000 100755 --- a/tools/chk_dup.py +++ b/tools/chk_dup.py @@ -8,9 +8,19 @@ import os import collections -def loadjsons(path): +def loadjsons(path, return_paths=False): """ - Find all Jsons and load them in a dict + Find all Jsons and load them in a dict + + Parameters: + path: string + return_names: boolean, if the name of the file should be returned, + default: False + + Returns: + List of parsed file contents. + If return_paths is True, then every list item is a tuple of the + file name and the file content """ files = [] data = [] @@ -18,9 +28,14 @@ def loadjsons(path): if os.path.isfile(os.path.join(path, name)) and name.endswith('.json'): files.append(name) for jfile in files: - data.append(json.load(open("%s/%s" % (path, jfile)))) + filepath = os.path.join(path, jfile) + if return_paths: + data.append((filepath, json.load(open(filepath)))) + else: + data.append(json.load(json.load(open(filepath)))) return data + if __name__ == '__main__': """ Iterate all name + synonyms @@ -33,19 +48,19 @@ if __name__ == '__main__': items = djson.get('values') for entry in items: name = entry.get('value').strip().lower() - counter[name]+=1 + counter[name] += 1 namespace.append([name, djson.get('name')]) try: for synonym in entry.get('meta').get('synonyms'): name = synonym.strip().lower() - counter[name]+=1 + counter[name] += 1 namespace.append([name, djson.get('name')]) except (AttributeError, TypeError): pass counter = dict(counter) for key, val in counter.items(): - if val>1: - print ("Warning duplicate %s" % key) + if val > 1: + print("Warning duplicate %s" % key) for item in namespace: - if item[0]==key: - print (item) + if item[0] == key: + print(item) diff --git a/tools/chk_empty_strings.py b/tools/chk_empty_strings.py new file mode 100755 index 0000000..1ccac24 --- /dev/null +++ b/tools/chk_empty_strings.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# coding=utf-8 +""" + Tools to find empty string entries in galaxies +""" +from .chk_dup import loadjsons +import sys + + +if __name__ == '__main__': + jsons = loadjsons("clusters", return_paths=True) + retval = 0 + for clustername, djson in jsons: + items = djson.get('values') + for entry in items: + name = entry.get('value') + for key, value in entry.get('meta', {}).items(): + if isinstance(value, list): + if '' in value: + retval = 1 + print("Empty string found in Cluster %r: values/%s/meta/%s" + "" % (clustername, name, key), + file=sys.stderr) + sys.exit(retval) diff --git a/validate_all.sh b/validate_all.sh index 7d1a842..f797c55 100755 --- a/validate_all.sh +++ b/validate_all.sh @@ -84,3 +84,6 @@ do fi echo '' done + +# check for empyt strings in clusters +python3 -m tools.chk_empty_strings