chg: Use simplejson when possible & gunicorn for production.

bgpranking/archive.py

@@ -8,6 +8,9 @@ from dateutil.relativedelta import relativedelta
 from collections import defaultdict
 import zipfile
 import logging
+try:
+    import simplejson as json
+except ImportError:
     import json
 
 from .libs.helpers import safe_create_dir, set_running, unset_running

bgpranking/config/modules/bambenekconsulting_feeds.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+try:
+    import simplejson as json
+except ImportError:
     import json
 import re
 import requests

bgpranking/dbinsert.py

@@ -31,7 +31,7 @@ class DatabaseInsert():
         while True:
             if shutdown_requested():
                 break
-            uuids = self.redis_sanitized.spop('to_insert', 1000)
+            uuids = self.redis_sanitized.spop('to_insert', 100)
             if not uuids:
                 break
             p = self.redis_sanitized.pipeline(transaction=False)

bgpranking/libs/helpers.py

@@ -9,6 +9,9 @@ from redis import StrictRedis
 from redis.exceptions import ConnectionError
 from datetime import datetime, timedelta
 import time
+try:
+    import simplejson as json
+except ImportError:
     import json
 from pyipasnhistory import IPASNHistory
 

bgpranking/libs/statsripe.py

@@ -7,6 +7,9 @@ from datetime import datetime, timedelta
 from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network
 from typing import TypeVar
 from .helpers import get_homedir, safe_create_dir
+try:
+    import simplejson as json
+except ImportError:
     import json
 from dateutil.parser import parse
 import copy

bgpranking/modulesfetcher.py

@@ -8,6 +8,9 @@ from hashlib import sha512  # Faster than sha256 on 64b machines.
 from pathlib import Path
 import logging
 from pid import PidFile, PidFileError
+try:
+    import simplejson as json
+except ImportError:
     import json
 
 from .libs.helpers import safe_create_dir, set_running, unset_running

bgpranking/monitor.py

@@ -1,5 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+try:
+    import simplejson as json
+except ImportError:
     import json
 
 from redis import StrictRedis

bgpranking/parser.py

@@ -4,6 +4,9 @@
 from datetime import datetime
 from pathlib import Path
 import logging
+try:
+    import simplejson as json
+except ImportError:
     import json
 import re
 from redis import StrictRedis

bgpranking/sanitizer.py

@@ -35,7 +35,7 @@ class Sanitizer():
         while True:
             if shutdown_requested():
                 break
-            uuids = self.redis_intake.spop('intake', 1000)
+            uuids = self.redis_intake.spop('intake', 100)
             if not uuids:
                 break
             for_cache = []

bgpranking/shadowserverfetcher.py

@@ -9,6 +9,9 @@ from typing import Tuple
 from datetime import datetime, date
 from pathlib import Path
 from .libs.helpers import safe_create_dir, set_running, unset_running
+try:
+    import simplejson as json
+except ImportError:
     import json
 
 

bin/ssfetcher.py

@@ -2,6 +2,9 @@
 # -*- coding: utf-8 -*-
 
 import logging
+try:
+    import simplejson as json
+except ImportError:
     import json
 import asyncio
 from pathlib import Path

requirements.txt

@@ -5,6 +5,7 @@ git+https://github.com/jsommers/pytricia.git
 git+https://github.com/trbs/pid.git
 aiohttp
 requests
+simplejson
 
 git+https://github.com/MISP/PyTaxonomies
 git+https://github.com/MISP/PyMISPGalaxies.git

website/readme.md

@@ -4,6 +4,6 @@ Install the dependencies, run
 
 ```bash
 export FLASK_APP=${BGPRANKING_HOME}/website/web/__init__.py
-flask run --port 5005
+gunicorn --worker-class gevent -w 10 -b 127.0.0.1:5005 web:app
 ```
 

website/requirements.txt

@@ -1,6 +1,8 @@
 # Web thing
 flask
 flask-bootstrap
+gunicorn[gevent]
+
 
 # Other
 pycountry

website/web/__init__.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+try:
+    import simplejson as json
+except ImportError:
     import json
 
 from flask import Flask, render_template, request, session, Response