chg: Use UNIX sockets instead of local TCP

bgpranking/dbinsert.py

@@ -2,18 +2,17 @@
 # -*- coding: utf-8 -*-
 
 import logging
-from redis import Redis
 from redis import StrictRedis
-from .libs.helpers import shutdown_requested, set_running, unset_running
+from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
 
 
 class DatabaseInsert():
 
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
-        self.ardb_storage = StrictRedis(host='localhost', port=16579, decode_responses=True)
-        self.redis_sanitized = Redis(host='localhost', port=6580, db=0, decode_responses=True)
-        self.ris_cache = Redis(host='localhost', port=6581, db=0, decode_responses=True)
+        self.ardb_storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
+        self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
+        self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
         self.logger.debug('Starting import')
 
     def __init_logger(self, loglevel):

bgpranking/libs/helpers.py

@@ -37,23 +37,43 @@ def safe_create_dir(to_create: Path):
 
 
 def set_running(name: str):
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     r.hset('running', name, 1)
 
 
 def unset_running(name: str):
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     r.hdel('running', name)
 
 
 def is_running():
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     return r.hgetall('running')
 
 
+def get_socket_path(name: str):
+    mapping = {
+        'ris': Path('cache', 'ris.sock'),
+        'prefixes': Path('cache', 'prefixes.sock'),
+        'storage': Path('storage', 'storage.sock'),
+        'intake': Path('temp', 'intake.sock'),
+        'prepare': Path('temp', 'prepare.sock'),
+    }
+    return str(get_homedir() / mapping[name])
+
+
+def check_running(name: str):
+    socket_path = get_socket_path(name)
+    try:
+        r = StrictRedis(unix_socket_path=socket_path)
+        return r.ping()
+    except ConnectionError:
+        return False
+
+
 def shutdown_requested():
     try:
-        r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+        r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
         return r.exists('shutdown')
     except ConnectionRefusedError:
         return True

bgpranking/monitor.py

@@ -2,17 +2,18 @@
 # -*- coding: utf-8 -*-
 
 from redis import StrictRedis
+from .libs.helpers import get_socket_path
 
 
 class Monitor():
 
     def __init__(self):
-        self.intake = StrictRedis(host='localhost', port=6579, db=0, decode_responses=True)
-        self.sanitize = StrictRedis(host='localhost', port=6580, db=0, decode_responses=True)
-        self.ris_cache = StrictRedis(host='localhost', port=6581, db=0, decode_responses=True)
-        self.prefix_cache = StrictRedis(host='localhost', port=6582, db=0, decode_responses=True)
-        self.running = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
-        self.storage = StrictRedis(host='localhost', port=16579, decode_responses=True)
+        self.intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
+        self.sanitize = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
+        self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
+        self.prefix_cache = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
+        self.running = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
+        self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
 
     def get_runinng(self):
         return self.running.hgetall('running')

bgpranking/parser.py

@@ -6,7 +6,7 @@ from pathlib import Path
 import logging
 import json
 import re
-from redis import Redis
+from redis import StrictRedis
 from uuid import uuid4
 from io import BytesIO
 import importlib
@@ -14,7 +14,7 @@ import importlib
 from typing import List
 import types
 
-from .libs.helpers import safe_create_dir, set_running, unset_running
+from .libs.helpers import safe_create_dir, set_running, unset_running, get_socket_path
 
 
 class RawFilesParser():
@@ -31,7 +31,7 @@ class RawFilesParser():
         self.directory = storage_directory / self.vendor / self.listname
         safe_create_dir(self.directory)
         self.__init_logger(loglevel)
-        self.redis_intake = Redis(host='localhost', port=6579, db=0)
+        self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0)
         self.logger.debug('Starting intake on {}'.format(self.source))
 
     def __init_logger(self, loglevel):

bgpranking/prefixdb.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import logging
-from redis import Redis
+from redis import StrictRedis
 from ipaddress import ip_network
 import requests
 import gzip
@@ -10,7 +10,7 @@ from io import BytesIO
 from collections import defaultdict
 import re
 import time
-from .libs.helpers import set_running, unset_running
+from .libs.helpers import set_running, unset_running, get_socket_path
 
 
 # Dataset source: Routeviews Prefix to AS mappings Dataset for IPv4 and IPv6
@@ -21,7 +21,7 @@ class PrefixDatabase():
 
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
-        self.prefix_cache = Redis(host='localhost', port=6582, db=0, decode_responses=True)
+        self.prefix_cache = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
         self.ipv6_url = 'http://data.caida.org/datasets/routing/routeviews6-prefix2as/{}'
         self.ipv4_url = 'http://data.caida.org/datasets/routing/routeviews-prefix2as/{}'
 

bgpranking/risfetcher.py

@@ -2,12 +2,12 @@
 # -*- coding: utf-8 -*-
 
 import logging
-from redis import Redis
+from redis import StrictRedis
 
 import time
 import pytricia
 import ipaddress
-from .libs.helpers import shutdown_requested, set_running, unset_running
+from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
 
 
 class RISPrefixLookup():
@@ -15,8 +15,8 @@ class RISPrefixLookup():
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
         self.logger.info('Starting RIS Prefix fetcher')
-        self.prefix_db = Redis(host='localhost', port=6582, db=0, decode_responses=True)
-        self.longest_prefix_matching = Redis(host='localhost', port=6581, db=0, decode_responses=True)
+        self.prefix_db = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
+        self.longest_prefix_matching = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
         self.tree_v4 = pytricia.PyTricia()
         self.tree_v6 = pytricia.PyTricia(128)
         self.init_tree()

bgpranking/sanitizer.py

@@ -4,7 +4,7 @@
 from dateutil import parser
 import logging
 from redis import StrictRedis
-from .libs.helpers import shutdown_requested, set_running, unset_running
+from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
 
 import ipaddress
 
@@ -13,9 +13,9 @@ class Sanitizer():
 
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
-        self.redis_intake = StrictRedis(host='localhost', port=6579, db=0, decode_responses=True)
-        self.redis_sanitized = StrictRedis(host='localhost', port=6580, db=0, decode_responses=True)
-        self.ris_cache = StrictRedis(host='localhost', port=6581, db=0, decode_responses=True)
+        self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
+        self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
+        self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
         self.logger.debug('Starting import')
 
     def __init_logger(self, loglevel):

bin/run_backend.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from bgpranking.libs.helpers import get_homedir
+from bgpranking.libs.helpers import get_homedir, check_running
 from subprocess import Popen
 import time
 from pathlib import Path
-from redis import Redis
-from redis.exceptions import ConnectionError
 
 import argparse
 
@@ -14,7 +12,7 @@ import argparse
 def launch_cache(storage_directory: Path=None):
     if not storage_directory:
         storage_directory = get_homedir()
-    if not check_running('127.0.0.1', 6581) and not check_running('127.0.0.1', 6582):
+    if not check_running('ris') and not check_running('prefixes'):
         Popen(["./run_redis.sh"], cwd=(storage_directory / 'cache'))
 
 
@@ -27,7 +25,7 @@ def shutdown_cache(storage_directory: Path=None):
 def launch_temp(storage_directory: Path=None):
     if not storage_directory:
         storage_directory = get_homedir()
-    if not check_running('127.0.0.1', 6579) and not check_running('127.0.0.1', 6580):
+    if not check_running('intake') and not check_running('prepare'):
         Popen(["./run_redis.sh"], cwd=(storage_directory / 'temp'))
 
 
@@ -40,7 +38,7 @@ def shutdown_temp(storage_directory: Path=None):
 def launch_storage(storage_directory: Path=None):
     if not storage_directory:
         storage_directory = get_homedir()
-    if not check_running('127.0.0.1', 16579):
+    if not check_running('storage'):
         Popen(["./run_ardb.sh"], cwd=(storage_directory / 'storage'))
 
 
@@ -50,14 +48,6 @@ def shutdown_storage(storage_directory: Path=None):
     Popen(["./shutdown_ardb.sh"], cwd=(storage_directory / 'storage'))
 
 
-def check_running(host, port):
-    try:
-        r = Redis(host=host, port=port)
-        return r.ping()
-    except ConnectionError:
-        return False
-
-
 def launch_all():
     launch_cache()
     launch_temp()
@@ -65,26 +55,25 @@ def launch_all():
 
 
 def check_all(stop=False):
-    backends = [['127.0.0.1', 6579, False], ['127.0.0.1', 6580, False],
-                ['127.0.0.1', 6581, False], ['127.0.0.1', 6582, False],
-                ['127.0.0.1', 16579, False]]
+    backends = [['ris', False], ['prefixes', False], ['storage', False],
+                ['intake', False], ['prepare', False]]
     while True:
         for b in backends:
             try:
-                b[2] = check_running(b[0], b[1])
+                b[1] = check_running(b[0])
             except Exception:
-                b[2] = False
+                b[1] = False
         if stop:
-            if not any(b[2] for b in backends):
+            if not any(b[1] for b in backends):
                 break
         else:
-            if all(b[2] for b in backends):
+            if all(b[1] for b in backends):
                 break
         for b in backends:
-            if not stop and not b[2]:
-                print('Waiting on {}:{}'.format(b[0], b[1]))
-            if stop and b[2]:
-                print('Waiting on {}:{}'.format(b[0], b[1]))
+            if not stop and not b[1]:
+                print('Waiting on {}'.format(b[0]))
+            if stop and b[1]:
+                print('Waiting on {}'.format(b[0]))
         time.sleep(1)
 
 

bin/shutdown.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from bgpranking.libs.helpers import is_running
+from bgpranking.libs.helpers import is_running, get_socket_path
 import time
 from redis import StrictRedis
 
 if __name__ == '__main__':
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     r.set('shutdown', 1)
     while True:
         running = is_running()

cache/prefixes.conf

@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6582
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket prefixes.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
@@ -168,7 +168,7 @@ loglevel notice
 # Specify the log file name. Also the empty string can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile "ranking.log"
+logfile "prefixes.log"
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
 # and optionally update the other syslog parameters to suit your needs.

cache/ris.conf

@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6581
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket ris.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0

cache/run_redis.sh

@@ -3,5 +3,5 @@
 set -e
 set -x
 
-../../redis/src/redis-server ./6581.conf
-../../redis/src/redis-server ./6582.conf
+../../redis/src/redis-server ./ris.conf
+../../redis/src/redis-server ./prefixes.conf

cache/shutdown_redis.sh

@@ -3,5 +3,5 @@
 # set -e
 set -x
 
-../../redis/src/redis-cli -p 6581 shutdown
-../../redis/src/redis-cli -p 6582 shutdown
+../../redis/src/redis-cli -s ./ris.sock shutdown
+../../redis/src/redis-cli -s ./prefixes.sock shutdown

storage/ardb.conf

@@ -26,13 +26,13 @@ pidfile ${ARDB_HOME}/ardb.pid
 thread-pool-size              4
 
 #Accept connections on the specified host&port/unix socket, default is 0.0.0.0:16379.
-server[0].listen              0.0.0.0:16579
+#server[0].listen              0.0.0.0:16579
 # If current qps exceed the limit, Ardb would return an error.
 #server[0].qps-limit           1000
 
 #listen on unix socket
-#server[1].listen             /tmp/ardb.sock
-#server[1].unixsocketperm     755
+server[0].listen             storage.sock
+server[0].unixsocketperm     755
 #server[1].qps-limit          1000
 
 # 'qps-limit-per-host' used to limit the request per second from same host

storage/shutdown_ardb.sh

@@ -3,4 +3,4 @@
 set -e
 set -x
 
-../../redis/src/redis-cli -p 16579 shutdown save
+../../redis/src/redis-cli -s ./storage.sock shutdown save

temp/intake.conf

@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6579
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket intake.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0

temp/prepare.conf

@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6580
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket prepare.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
@@ -250,7 +250,7 @@ rdbcompression yes
 rdbchecksum yes
 
 # The filename where to dump the DB
-dbfilename pre_insert.rdb
+dbfilename prepare.rdb
 
 # The working directory.
 #

temp/run_redis.sh

@@ -3,5 +3,5 @@
 set -e
 set -x
 
-../../redis/src/redis-server ./6579.conf
-../../redis/src/redis-server ./6580.conf
+../../redis/src/redis-server ./intake.conf
+../../redis/src/redis-server ./prepare.conf

temp/shutdown_redis.sh

@@ -3,5 +3,5 @@
 # set -e
 set -x
 
-../../redis/src/redis-cli -p 6579 shutdown
-../../redis/src/redis-cli -p 6580 shutdown
+../../redis/src/redis-cli -s ./intake.sock shutdown
+../../redis/src/redis-cli -s ./prepare.sock shutdown