Merge branch 'main' of github.com:MISP/misp-warninglists into main

.github/workflows/nosetests.yml

@@ -0,0 +1,39 @@
+name: Python application
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{matrix.python-version}}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{matrix.python-version}}
+
+    - name: Initialize submodules
+      run: git submodule update --init --recursive
+
+    - name: Install system dependencies
+      run: |
+        sudo apt install jq moreutils
+
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade jsonschema
+
+    - name: Test
+      run: |
+        ./validate_all.sh

tools/make_list_unique.py

@@ -7,7 +7,7 @@ import json
 
 for p in Path('../lists/').glob('*/*.json'):
     with p.open() as _f:
-        warninglist = json.load(_f, encoding="utf-8")
+        warninglist = json.load(_f)
     warninglist['list'] = sorted(list(set(warninglist['list'])))
 
     with p.open('w') as _f:

tools/validate_values.py

@@ -54,7 +54,7 @@ def validate(values: List[str], func) -> Iterator[Tuple[str, ValueError]]:
 def validate_file(p: Path) -> Iterator[InvalidListValue]:
     invalid_values = []
     with p.open() as f:
-        warninglist = json.load(f, encoding="utf-8")
+        warninglist = json.load(f)
         if warninglist["type"] == "cidr":
             invalid_values = validate(warninglist["list"], lambda value: ip_network(value, strict=True))
         elif warninglist["type"] == "regexp":