Browse Source

Add test for empty strings

Should prevent MISP/misp-galaxy#438
pull/446/head
Sebastian Wagner 1 year ago
parent
commit
c93103bba1
No known key found for this signature in database GPG Key ID: 4BB5A67FBECFA8A5
5 changed files with 52 additions and 9 deletions
  1. +1
    -0
      .gitignore
  2. +0
    -0
      tools/__init__.py
  3. +24
    -9
      tools/chk_dup.py
  4. +24
    -0
      tools/chk_empty_strings.py
  5. +3
    -0
      validate_all.sh

+ 1
- 0
.gitignore View File

@@ -0,0 +1 @@
__pycache__

+ 0
- 0
tools/__init__.py View File


+ 24
- 9
tools/chk_dup.py View File

@@ -8,9 +8,19 @@ import os
import collections


def loadjsons(path):
def loadjsons(path, return_paths=False):
"""
Find all Jsons and load them in a dict
Find all Jsons and load them in a dict

Parameters:
path: string
return_names: boolean, if the name of the file should be returned,
default: False

Returns:
List of parsed file contents.
If return_paths is True, then every list item is a tuple of the
file name and the file content
"""
files = []
data = []
@@ -18,9 +28,14 @@ def loadjsons(path):
if os.path.isfile(os.path.join(path, name)) and name.endswith('.json'):
files.append(name)
for jfile in files:
data.append(json.load(open("%s/%s" % (path, jfile))))
filepath = os.path.join(path, jfile)
if return_paths:
data.append((filepath, json.load(open(filepath))))
else:
data.append(json.load(json.load(open(filepath))))
return data


if __name__ == '__main__':
"""
Iterate all name + synonyms
@@ -33,19 +48,19 @@ if __name__ == '__main__':
items = djson.get('values')
for entry in items:
name = entry.get('value').strip().lower()
counter[name]+=1
counter[name] += 1
namespace.append([name, djson.get('name')])
try:
for synonym in entry.get('meta').get('synonyms'):
name = synonym.strip().lower()
counter[name]+=1
counter[name] += 1
namespace.append([name, djson.get('name')])
except (AttributeError, TypeError):
pass
counter = dict(counter)
for key, val in counter.items():
if val>1:
print ("Warning duplicate %s" % key)
if val > 1:
print("Warning duplicate %s" % key)
for item in namespace:
if item[0]==key:
print (item)
if item[0] == key:
print(item)

+ 24
- 0
tools/chk_empty_strings.py View File

@@ -0,0 +1,24 @@
#!/usr/bin/env python3
# coding=utf-8
"""
Tools to find empty string entries in galaxies
"""
from .chk_dup import loadjsons
import sys


if __name__ == '__main__':
jsons = loadjsons("clusters", return_paths=True)
retval = 0
for clustername, djson in jsons:
items = djson.get('values')
for entry in items:
name = entry.get('value')
for key, value in entry.get('meta', {}).items():
if isinstance(value, list):
if '' in value:
retval = 1
print("Empty string found in Cluster %r: values/%s/meta/%s"
"" % (clustername, name, key),
file=sys.stderr)
sys.exit(retval)

+ 3
- 0
validate_all.sh View File

@@ -84,3 +84,6 @@ do
fi
echo ''
done

# check for empyt strings in clusters
python3 -m tools.chk_empty_strings

Loading…
Cancel
Save