diff --git a/bgpranking/archive.py b/bgpranking/archive.py index ad3e042..5e75643 100644 --- a/bgpranking/archive.py +++ b/bgpranking/archive.py @@ -8,7 +8,10 @@ from dateutil.relativedelta import relativedelta from collections import defaultdict import zipfile import logging -import json +try: + import simplejson as json +except ImportError: + import json from .libs.helpers import safe_create_dir, set_running, unset_running diff --git a/bgpranking/config/modules/bambenekconsulting_feeds.py b/bgpranking/config/modules/bambenekconsulting_feeds.py index bc2f03d..bfc7336 100644 --- a/bgpranking/config/modules/bambenekconsulting_feeds.py +++ b/bgpranking/config/modules/bambenekconsulting_feeds.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import json +try: + import simplejson as json +except ImportError: + import json import re import requests from bs4 import BeautifulSoup diff --git a/bgpranking/dbinsert.py b/bgpranking/dbinsert.py index 9106432..0e168fa 100644 --- a/bgpranking/dbinsert.py +++ b/bgpranking/dbinsert.py @@ -31,7 +31,7 @@ class DatabaseInsert(): while True: if shutdown_requested(): break - uuids = self.redis_sanitized.spop('to_insert', 1000) + uuids = self.redis_sanitized.spop('to_insert', 100) if not uuids: break p = self.redis_sanitized.pipeline(transaction=False) diff --git a/bgpranking/libs/helpers.py b/bgpranking/libs/helpers.py index 2ebaea6..dda2c74 100644 --- a/bgpranking/libs/helpers.py +++ b/bgpranking/libs/helpers.py @@ -9,7 +9,10 @@ from redis import StrictRedis from redis.exceptions import ConnectionError from datetime import datetime, timedelta import time -import json +try: + import simplejson as json +except ImportError: + import json from pyipasnhistory import IPASNHistory diff --git a/bgpranking/libs/statsripe.py b/bgpranking/libs/statsripe.py index 5c10a45..10f8766 100644 --- a/bgpranking/libs/statsripe.py +++ b/bgpranking/libs/statsripe.py @@ -7,7 +7,10 @@ from datetime import datetime, timedelta from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network from typing import TypeVar from .helpers import get_homedir, safe_create_dir -import json +try: + import simplejson as json +except ImportError: + import json from dateutil.parser import parse import copy diff --git a/bgpranking/modulesfetcher.py b/bgpranking/modulesfetcher.py index 4f8a8d7..4f6212c 100644 --- a/bgpranking/modulesfetcher.py +++ b/bgpranking/modulesfetcher.py @@ -8,7 +8,10 @@ from hashlib import sha512 # Faster than sha256 on 64b machines. from pathlib import Path import logging from pid import PidFile, PidFileError -import json +try: + import simplejson as json +except ImportError: + import json from .libs.helpers import safe_create_dir, set_running, unset_running diff --git a/bgpranking/monitor.py b/bgpranking/monitor.py index fd144eb..a34ed74 100644 --- a/bgpranking/monitor.py +++ b/bgpranking/monitor.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json +try: + import simplejson as json +except ImportError: + import json from redis import StrictRedis from .libs.helpers import get_socket_path, get_ipasn diff --git a/bgpranking/parser.py b/bgpranking/parser.py index 3c92ef3..444a42b 100644 --- a/bgpranking/parser.py +++ b/bgpranking/parser.py @@ -4,7 +4,10 @@ from datetime import datetime from pathlib import Path import logging -import json +try: + import simplejson as json +except ImportError: + import json import re from redis import StrictRedis from uuid import uuid4 diff --git a/bgpranking/sanitizer.py b/bgpranking/sanitizer.py index 53476e4..3f504c2 100644 --- a/bgpranking/sanitizer.py +++ b/bgpranking/sanitizer.py @@ -35,7 +35,7 @@ class Sanitizer(): while True: if shutdown_requested(): break - uuids = self.redis_intake.spop('intake', 1000) + uuids = self.redis_intake.spop('intake', 100) if not uuids: break for_cache = [] diff --git a/bgpranking/shadowserverfetcher.py b/bgpranking/shadowserverfetcher.py index 54361ff..60cfe02 100644 --- a/bgpranking/shadowserverfetcher.py +++ b/bgpranking/shadowserverfetcher.py @@ -9,7 +9,10 @@ from typing import Tuple from datetime import datetime, date from pathlib import Path from .libs.helpers import safe_create_dir, set_running, unset_running -import json +try: + import simplejson as json +except ImportError: + import json class ShadowServerFetcher(): diff --git a/bin/ssfetcher.py b/bin/ssfetcher.py index 2930337..c075d52 100755 --- a/bin/ssfetcher.py +++ b/bin/ssfetcher.py @@ -2,7 +2,10 @@ # -*- coding: utf-8 -*- import logging -import json +try: + import simplejson as json +except ImportError: + import json import asyncio from pathlib import Path import aiohttp diff --git a/requirements.txt b/requirements.txt index b82c1ce..ad73b83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ git+https://github.com/jsommers/pytricia.git git+https://github.com/trbs/pid.git aiohttp requests +simplejson git+https://github.com/MISP/PyTaxonomies git+https://github.com/MISP/PyMISPGalaxies.git diff --git a/website/readme.md b/website/readme.md index 06313f5..6160647 100644 --- a/website/readme.md +++ b/website/readme.md @@ -4,6 +4,6 @@ Install the dependencies, run ```bash export FLASK_APP=${BGPRANKING_HOME}/website/web/__init__.py -flask run --port 5005 +gunicorn --worker-class gevent -w 10 -b 127.0.0.1:5005 web:app ``` diff --git a/website/requirements.txt b/website/requirements.txt index b2d5f6a..87c6847 100644 --- a/website/requirements.txt +++ b/website/requirements.txt @@ -1,6 +1,8 @@ # Web thing flask flask-bootstrap +gunicorn[gevent] + # Other pycountry diff --git a/website/web/__init__.py b/website/web/__init__.py index 763e275..1c0116b 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1,7 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json +try: + import simplejson as json +except ImportError: + import json from flask import Flask, render_template, request, session, Response from flask_bootstrap import Bootstrap