chg: Use simplejson when possible & gunicorn for production.

bgpranking/archive.py

@@ -8,7 +8,10 @@ from dateutil.relativedelta import relativedelta
 from collections import defaultdict
 import zipfile
 import logging
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 from .libs.helpers import safe_create_dir, set_running, unset_running
 

bgpranking/config/modules/bambenekconsulting_feeds.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 import re
 import requests
 from bs4 import BeautifulSoup

bgpranking/dbinsert.py

@@ -31,7 +31,7 @@ class DatabaseInsert():
         while True:
             if shutdown_requested():
                 break
-            uuids = self.redis_sanitized.spop('to_insert', 1000)
+            uuids = self.redis_sanitized.spop('to_insert', 100)
             if not uuids:
                 break
             p = self.redis_sanitized.pipeline(transaction=False)

bgpranking/libs/helpers.py

@@ -9,7 +9,10 @@ from redis import StrictRedis
 from redis.exceptions import ConnectionError
 from datetime import datetime, timedelta
 import time
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 from pyipasnhistory import IPASNHistory
 
 

bgpranking/libs/statsripe.py

@@ -7,7 +7,10 @@ from datetime import datetime, timedelta
 from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network
 from typing import TypeVar
 from .helpers import get_homedir, safe_create_dir
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 from dateutil.parser import parse
 import copy
 

bgpranking/modulesfetcher.py

@@ -8,7 +8,10 @@ from hashlib import sha512  # Faster than sha256 on 64b machines.
 from pathlib import Path
 import logging
 from pid import PidFile, PidFileError
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 from .libs.helpers import safe_create_dir, set_running, unset_running
 

bgpranking/monitor.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 from redis import StrictRedis
 from .libs.helpers import get_socket_path, get_ipasn

bgpranking/parser.py

@@ -4,7 +4,10 @@
 from datetime import datetime
 from pathlib import Path
 import logging
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 import re
 from redis import StrictRedis
 from uuid import uuid4

bgpranking/sanitizer.py

@@ -35,7 +35,7 @@ class Sanitizer():
         while True:
             if shutdown_requested():
                 break
-            uuids = self.redis_intake.spop('intake', 1000)
+            uuids = self.redis_intake.spop('intake', 100)
             if not uuids:
                 break
             for_cache = []

bgpranking/shadowserverfetcher.py

@@ -9,7 +9,10 @@ from typing import Tuple
 from datetime import datetime, date
 from pathlib import Path
 from .libs.helpers import safe_create_dir, set_running, unset_running
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 
 class ShadowServerFetcher():

bin/ssfetcher.py

@@ -2,7 +2,10 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 import asyncio
 from pathlib import Path
 import aiohttp

requirements.txt

@@ -5,6 +5,7 @@ git+https://github.com/jsommers/pytricia.git
 git+https://github.com/trbs/pid.git
 aiohttp
 requests
+simplejson
 
 git+https://github.com/MISP/PyTaxonomies
 git+https://github.com/MISP/PyMISPGalaxies.git

website/readme.md

@@ -4,6 +4,6 @@ Install the dependencies, run
 
 ```bash
 export FLASK_APP=${BGPRANKING_HOME}/website/web/__init__.py
-flask run --port 5005
+gunicorn --worker-class gevent -w 10 -b 127.0.0.1:5005 web:app
 ```
 

website/requirements.txt

@@ -1,6 +1,8 @@
 # Web thing
 flask
 flask-bootstrap
+gunicorn[gevent]
+
 
 # Other
 pycountry

website/web/__init__.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import json
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 from flask import Flask, render_template, request, session, Response
 from flask_bootstrap import Bootstrap