From 8180d1e0f33051bddbe8efb43c56e0e973a5f415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 5 Apr 2018 14:36:01 +0200
Subject: [PATCH] chg: Use UNIX sockets instead of local TCP

---
 bgpranking/dbinsert.py             |  9 +++----
 bgpranking/libs/helpers.py         | 28 ++++++++++++++++++---
 bgpranking/monitor.py              | 13 +++++-----
 bgpranking/parser.py               |  6 ++---
 bgpranking/prefixdb.py             |  6 ++---
 bgpranking/risfetcher.py           |  8 +++---
 bgpranking/sanitizer.py            |  8 +++---
 bin/run_backend.py                 | 39 +++++++++++-------------------
 bin/shutdown.py                    |  4 +--
 cache/{6582.conf => prefixes.conf} |  8 +++---
 cache/{6581.conf => ris.conf}      |  6 ++---
 cache/run_redis.sh                 |  4 +--
 cache/shutdown_redis.sh            |  4 +--
 storage/ardb.conf                  |  6 ++---
 storage/shutdown_ardb.sh           |  2 +-
 temp/{6579.conf => intake.conf}    |  6 ++---
 temp/{6580.conf => prepare.conf}   |  8 +++---
 temp/run_redis.sh                  |  4 +--
 temp/shutdown_redis.sh             |  4 +--
 19 files changed, 91 insertions(+), 82 deletions(-)
 rename cache/{6582.conf => prefixes.conf} (99%)
 rename cache/{6581.conf => ris.conf} (99%)
 rename temp/{6579.conf => intake.conf} (99%)
 rename temp/{6580.conf => prepare.conf} (99%)

diff --git a/bgpranking/dbinsert.py b/bgpranking/dbinsert.py
index 3f57edd..e51bc86 100644
--- a/bgpranking/dbinsert.py
+++ b/bgpranking/dbinsert.py
@@ -2,18 +2,17 @@
 # -*- coding: utf-8 -*-
 
 import logging
-from redis import Redis
 from redis import StrictRedis
-from .libs.helpers import shutdown_requested, set_running, unset_running
+from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
 
 
 class DatabaseInsert():
 
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
-        self.ardb_storage = StrictRedis(host='localhost', port=16579, decode_responses=True)
-        self.redis_sanitized = Redis(host='localhost', port=6580, db=0, decode_responses=True)
-        self.ris_cache = Redis(host='localhost', port=6581, db=0, decode_responses=True)
+        self.ardb_storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
+        self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
+        self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
         self.logger.debug('Starting import')
 
     def __init_logger(self, loglevel):
diff --git a/bgpranking/libs/helpers.py b/bgpranking/libs/helpers.py
index 2e5aa00..2b65f30 100644
--- a/bgpranking/libs/helpers.py
+++ b/bgpranking/libs/helpers.py
@@ -37,23 +37,43 @@ def safe_create_dir(to_create: Path):
 
 
 def set_running(name: str):
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     r.hset('running', name, 1)
 
 
 def unset_running(name: str):
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     r.hdel('running', name)
 
 
 def is_running():
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     return r.hgetall('running')
 
 
+def get_socket_path(name: str):
+    mapping = {
+        'ris': Path('cache', 'ris.sock'),
+        'prefixes': Path('cache', 'prefixes.sock'),
+        'storage': Path('storage', 'storage.sock'),
+        'intake': Path('temp', 'intake.sock'),
+        'prepare': Path('temp', 'prepare.sock'),
+    }
+    return str(get_homedir() / mapping[name])
+
+
+def check_running(name: str):
+    socket_path = get_socket_path(name)
+    try:
+        r = StrictRedis(unix_socket_path=socket_path)
+        return r.ping()
+    except ConnectionError:
+        return False
+
+
 def shutdown_requested():
     try:
-        r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+        r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
         return r.exists('shutdown')
     except ConnectionRefusedError:
         return True
diff --git a/bgpranking/monitor.py b/bgpranking/monitor.py
index adaded1..64ddfaa 100644
--- a/bgpranking/monitor.py
+++ b/bgpranking/monitor.py
@@ -2,17 +2,18 @@
 # -*- coding: utf-8 -*-
 
 from redis import StrictRedis
+from .libs.helpers import get_socket_path
 
 
 class Monitor():
 
     def __init__(self):
-        self.intake = StrictRedis(host='localhost', port=6579, db=0, decode_responses=True)
-        self.sanitize = StrictRedis(host='localhost', port=6580, db=0, decode_responses=True)
-        self.ris_cache = StrictRedis(host='localhost', port=6581, db=0, decode_responses=True)
-        self.prefix_cache = StrictRedis(host='localhost', port=6582, db=0, decode_responses=True)
-        self.running = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
-        self.storage = StrictRedis(host='localhost', port=16579, decode_responses=True)
+        self.intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
+        self.sanitize = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
+        self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
+        self.prefix_cache = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
+        self.running = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
+        self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
 
     def get_runinng(self):
         return self.running.hgetall('running')
diff --git a/bgpranking/parser.py b/bgpranking/parser.py
index dae8156..56e4f16 100644
--- a/bgpranking/parser.py
+++ b/bgpranking/parser.py
@@ -6,7 +6,7 @@ from pathlib import Path
 import logging
 import json
 import re
-from redis import Redis
+from redis import StrictRedis
 from uuid import uuid4
 from io import BytesIO
 import importlib
@@ -14,7 +14,7 @@ import importlib
 from typing import List
 import types
 
-from .libs.helpers import safe_create_dir, set_running, unset_running
+from .libs.helpers import safe_create_dir, set_running, unset_running, get_socket_path
 
 
 class RawFilesParser():
@@ -31,7 +31,7 @@ class RawFilesParser():
         self.directory = storage_directory / self.vendor / self.listname
         safe_create_dir(self.directory)
         self.__init_logger(loglevel)
-        self.redis_intake = Redis(host='localhost', port=6579, db=0)
+        self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0)
         self.logger.debug('Starting intake on {}'.format(self.source))
 
     def __init_logger(self, loglevel):
diff --git a/bgpranking/prefixdb.py b/bgpranking/prefixdb.py
index 8057899..a15745b 100644
--- a/bgpranking/prefixdb.py
+++ b/bgpranking/prefixdb.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import logging
-from redis import Redis
+from redis import StrictRedis
 from ipaddress import ip_network
 import requests
 import gzip
@@ -10,7 +10,7 @@ from io import BytesIO
 from collections import defaultdict
 import re
 import time
-from .libs.helpers import set_running, unset_running
+from .libs.helpers import set_running, unset_running, get_socket_path
 
 
 # Dataset source: Routeviews Prefix to AS mappings Dataset for IPv4 and IPv6
@@ -21,7 +21,7 @@ class PrefixDatabase():
 
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
-        self.prefix_cache = Redis(host='localhost', port=6582, db=0, decode_responses=True)
+        self.prefix_cache = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
         self.ipv6_url = 'http://data.caida.org/datasets/routing/routeviews6-prefix2as/{}'
         self.ipv4_url = 'http://data.caida.org/datasets/routing/routeviews-prefix2as/{}'
 
diff --git a/bgpranking/risfetcher.py b/bgpranking/risfetcher.py
index 8953c91..d895c00 100644
--- a/bgpranking/risfetcher.py
+++ b/bgpranking/risfetcher.py
@@ -2,12 +2,12 @@
 # -*- coding: utf-8 -*-
 
 import logging
-from redis import Redis
+from redis import StrictRedis
 
 import time
 import pytricia
 import ipaddress
-from .libs.helpers import shutdown_requested, set_running, unset_running
+from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
 
 
 class RISPrefixLookup():
@@ -15,8 +15,8 @@ class RISPrefixLookup():
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
         self.logger.info('Starting RIS Prefix fetcher')
-        self.prefix_db = Redis(host='localhost', port=6582, db=0, decode_responses=True)
-        self.longest_prefix_matching = Redis(host='localhost', port=6581, db=0, decode_responses=True)
+        self.prefix_db = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
+        self.longest_prefix_matching = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
         self.tree_v4 = pytricia.PyTricia()
         self.tree_v6 = pytricia.PyTricia(128)
         self.init_tree()
diff --git a/bgpranking/sanitizer.py b/bgpranking/sanitizer.py
index 929513d..be7709f 100644
--- a/bgpranking/sanitizer.py
+++ b/bgpranking/sanitizer.py
@@ -4,7 +4,7 @@
 from dateutil import parser
 import logging
 from redis import StrictRedis
-from .libs.helpers import shutdown_requested, set_running, unset_running
+from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
 
 import ipaddress
 
@@ -13,9 +13,9 @@ class Sanitizer():
 
     def __init__(self, loglevel: int=logging.DEBUG):
         self.__init_logger(loglevel)
-        self.redis_intake = StrictRedis(host='localhost', port=6579, db=0, decode_responses=True)
-        self.redis_sanitized = StrictRedis(host='localhost', port=6580, db=0, decode_responses=True)
-        self.ris_cache = StrictRedis(host='localhost', port=6581, db=0, decode_responses=True)
+        self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
+        self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
+        self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
         self.logger.debug('Starting import')
 
     def __init_logger(self, loglevel):
diff --git a/bin/run_backend.py b/bin/run_backend.py
index 0a0e516..54e86d0 100755
--- a/bin/run_backend.py
+++ b/bin/run_backend.py
@@ -1,12 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from bgpranking.libs.helpers import get_homedir
+from bgpranking.libs.helpers import get_homedir, check_running
 from subprocess import Popen
 import time
 from pathlib import Path
-from redis import Redis
-from redis.exceptions import ConnectionError
 
 import argparse
 
@@ -14,7 +12,7 @@ import argparse
 def launch_cache(storage_directory: Path=None):
     if not storage_directory:
         storage_directory = get_homedir()
-    if not check_running('127.0.0.1', 6581) and not check_running('127.0.0.1', 6582):
+    if not check_running('ris') and not check_running('prefixes'):
         Popen(["./run_redis.sh"], cwd=(storage_directory / 'cache'))
 
 
@@ -27,7 +25,7 @@ def shutdown_cache(storage_directory: Path=None):
 def launch_temp(storage_directory: Path=None):
     if not storage_directory:
         storage_directory = get_homedir()
-    if not check_running('127.0.0.1', 6579) and not check_running('127.0.0.1', 6580):
+    if not check_running('intake') and not check_running('prepare'):
         Popen(["./run_redis.sh"], cwd=(storage_directory / 'temp'))
 
 
@@ -40,7 +38,7 @@ def shutdown_temp(storage_directory: Path=None):
 def launch_storage(storage_directory: Path=None):
     if not storage_directory:
         storage_directory = get_homedir()
-    if not check_running('127.0.0.1', 16579):
+    if not check_running('storage'):
         Popen(["./run_ardb.sh"], cwd=(storage_directory / 'storage'))
 
 
@@ -50,14 +48,6 @@ def shutdown_storage(storage_directory: Path=None):
     Popen(["./shutdown_ardb.sh"], cwd=(storage_directory / 'storage'))
 
 
-def check_running(host, port):
-    try:
-        r = Redis(host=host, port=port)
-        return r.ping()
-    except ConnectionError:
-        return False
-
-
 def launch_all():
     launch_cache()
     launch_temp()
@@ -65,26 +55,25 @@ def launch_all():
 
 
 def check_all(stop=False):
-    backends = [['127.0.0.1', 6579, False], ['127.0.0.1', 6580, False],
-                ['127.0.0.1', 6581, False], ['127.0.0.1', 6582, False],
-                ['127.0.0.1', 16579, False]]
+    backends = [['ris', False], ['prefixes', False], ['storage', False],
+                ['intake', False], ['prepare', False]]
     while True:
         for b in backends:
             try:
-                b[2] = check_running(b[0], b[1])
+                b[1] = check_running(b[0])
             except Exception:
-                b[2] = False
+                b[1] = False
         if stop:
-            if not any(b[2] for b in backends):
+            if not any(b[1] for b in backends):
                 break
         else:
-            if all(b[2] for b in backends):
+            if all(b[1] for b in backends):
                 break
         for b in backends:
-            if not stop and not b[2]:
-                print('Waiting on {}:{}'.format(b[0], b[1]))
-            if stop and b[2]:
-                print('Waiting on {}:{}'.format(b[0], b[1]))
+            if not stop and not b[1]:
+                print('Waiting on {}'.format(b[0]))
+            if stop and b[1]:
+                print('Waiting on {}'.format(b[0]))
         time.sleep(1)
 
 
diff --git a/bin/shutdown.py b/bin/shutdown.py
index 6b216c6..5454851 100755
--- a/bin/shutdown.py
+++ b/bin/shutdown.py
@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from bgpranking.libs.helpers import is_running
+from bgpranking.libs.helpers import is_running, get_socket_path
 import time
 from redis import StrictRedis
 
 if __name__ == '__main__':
-    r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
+    r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
     r.set('shutdown', 1)
     while True:
         running = is_running()
diff --git a/cache/6582.conf b/cache/prefixes.conf
similarity index 99%
rename from cache/6582.conf
rename to cache/prefixes.conf
index d81eea2..b2b7a6c 100644
--- a/cache/6582.conf
+++ b/cache/prefixes.conf
@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6582
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket prefixes.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
@@ -168,7 +168,7 @@ loglevel notice
 # Specify the log file name. Also the empty string can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile "ranking.log"
+logfile "prefixes.log"
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
 # and optionally update the other syslog parameters to suit your needs.
diff --git a/cache/6581.conf b/cache/ris.conf
similarity index 99%
rename from cache/6581.conf
rename to cache/ris.conf
index 1bb1172..7677d99 100644
--- a/cache/6581.conf
+++ b/cache/ris.conf
@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6581
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket ris.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
diff --git a/cache/run_redis.sh b/cache/run_redis.sh
index ee4600e..b32bbdc 100755
--- a/cache/run_redis.sh
+++ b/cache/run_redis.sh
@@ -3,5 +3,5 @@
 set -e
 set -x
 
-../../redis/src/redis-server ./6581.conf
-../../redis/src/redis-server ./6582.conf
+../../redis/src/redis-server ./ris.conf
+../../redis/src/redis-server ./prefixes.conf
diff --git a/cache/shutdown_redis.sh b/cache/shutdown_redis.sh
index 40934d1..9c45af7 100755
--- a/cache/shutdown_redis.sh
+++ b/cache/shutdown_redis.sh
@@ -3,5 +3,5 @@
 # set -e
 set -x
 
-../../redis/src/redis-cli -p 6581 shutdown
-../../redis/src/redis-cli -p 6582 shutdown
+../../redis/src/redis-cli -s ./ris.sock shutdown
+../../redis/src/redis-cli -s ./prefixes.sock shutdown
diff --git a/storage/ardb.conf b/storage/ardb.conf
index 000d93e..435354b 100644
--- a/storage/ardb.conf
+++ b/storage/ardb.conf
@@ -26,13 +26,13 @@ pidfile ${ARDB_HOME}/ardb.pid
 thread-pool-size              4
 
 #Accept connections on the specified host&port/unix socket, default is 0.0.0.0:16379.
-server[0].listen              0.0.0.0:16579
+#server[0].listen              0.0.0.0:16579
 # If current qps exceed the limit, Ardb would return an error.
 #server[0].qps-limit           1000
 
 #listen on unix socket
-#server[1].listen             /tmp/ardb.sock
-#server[1].unixsocketperm     755
+server[0].listen             storage.sock
+server[0].unixsocketperm     755
 #server[1].qps-limit          1000
 
 # 'qps-limit-per-host' used to limit the request per second from same host
diff --git a/storage/shutdown_ardb.sh b/storage/shutdown_ardb.sh
index b17ee6d..d888a20 100755
--- a/storage/shutdown_ardb.sh
+++ b/storage/shutdown_ardb.sh
@@ -3,4 +3,4 @@
 set -e
 set -x
 
-../../redis/src/redis-cli -p 16579 shutdown save
+../../redis/src/redis-cli -s ./storage.sock shutdown save
diff --git a/temp/6579.conf b/temp/intake.conf
similarity index 99%
rename from temp/6579.conf
rename to temp/intake.conf
index 70e810b..a678f2f 100644
--- a/temp/6579.conf
+++ b/temp/intake.conf
@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6579
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket intake.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
diff --git a/temp/6580.conf b/temp/prepare.conf
similarity index 99%
rename from temp/6580.conf
rename to temp/prepare.conf
index e5f5d94..48f22e7 100644
--- a/temp/6580.conf
+++ b/temp/prepare.conf
@@ -89,7 +89,7 @@ protected-mode yes
 
 # Accept connections on the specified port, default is 6379 (IANA #815344).
 # If port 0 is specified Redis will not listen on a TCP socket.
-port 6580
+port 0
 
 # TCP listen() backlog.
 #
@@ -106,8 +106,8 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-# unixsocketperm 700
+unixsocket prepare.sock
+unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
@@ -250,7 +250,7 @@ rdbcompression yes
 rdbchecksum yes
 
 # The filename where to dump the DB
-dbfilename pre_insert.rdb
+dbfilename prepare.rdb
 
 # The working directory.
 #
diff --git a/temp/run_redis.sh b/temp/run_redis.sh
index 491b9bf..5eb1308 100755
--- a/temp/run_redis.sh
+++ b/temp/run_redis.sh
@@ -3,5 +3,5 @@
 set -e
 set -x
 
-../../redis/src/redis-server ./6579.conf
-../../redis/src/redis-server ./6580.conf
+../../redis/src/redis-server ./intake.conf
+../../redis/src/redis-server ./prepare.conf
diff --git a/temp/shutdown_redis.sh b/temp/shutdown_redis.sh
index a40ab97..bb218dc 100755
--- a/temp/shutdown_redis.sh
+++ b/temp/shutdown_redis.sh
@@ -3,5 +3,5 @@
 # set -e
 set -x
 
-../../redis/src/redis-cli -p 6579 shutdown
-../../redis/src/redis-cli -p 6580 shutdown
+../../redis/src/redis-cli -s ./intake.sock shutdown
+../../redis/src/redis-cli -s ./prepare.sock shutdown