chg: Use UNIX sockets instead of local TCP

pull/12/head
Raphaël Vinot 2018-04-05 14:36:01 +02:00
parent c2b92c56eb
commit 8180d1e0f3
19 changed files with 91 additions and 82 deletions

View File

@ -2,18 +2,17 @@
# -*- coding: utf-8 -*-
import logging
from redis import Redis
from redis import StrictRedis
from .libs.helpers import shutdown_requested, set_running, unset_running
from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
class DatabaseInsert():
def __init__(self, loglevel: int=logging.DEBUG):
self.__init_logger(loglevel)
self.ardb_storage = StrictRedis(host='localhost', port=16579, decode_responses=True)
self.redis_sanitized = Redis(host='localhost', port=6580, db=0, decode_responses=True)
self.ris_cache = Redis(host='localhost', port=6581, db=0, decode_responses=True)
self.ardb_storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
self.logger.debug('Starting import')
def __init_logger(self, loglevel):

View File

@ -37,23 +37,43 @@ def safe_create_dir(to_create: Path):
def set_running(name: str):
r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
r.hset('running', name, 1)
def unset_running(name: str):
r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
r.hdel('running', name)
def is_running():
r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
return r.hgetall('running')
def get_socket_path(name: str):
mapping = {
'ris': Path('cache', 'ris.sock'),
'prefixes': Path('cache', 'prefixes.sock'),
'storage': Path('storage', 'storage.sock'),
'intake': Path('temp', 'intake.sock'),
'prepare': Path('temp', 'prepare.sock'),
}
return str(get_homedir() / mapping[name])
def check_running(name: str):
socket_path = get_socket_path(name)
try:
r = StrictRedis(unix_socket_path=socket_path)
return r.ping()
except ConnectionError:
return False
def shutdown_requested():
try:
r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
return r.exists('shutdown')
except ConnectionRefusedError:
return True

View File

@ -2,17 +2,18 @@
# -*- coding: utf-8 -*-
from redis import StrictRedis
from .libs.helpers import get_socket_path
class Monitor():
def __init__(self):
self.intake = StrictRedis(host='localhost', port=6579, db=0, decode_responses=True)
self.sanitize = StrictRedis(host='localhost', port=6580, db=0, decode_responses=True)
self.ris_cache = StrictRedis(host='localhost', port=6581, db=0, decode_responses=True)
self.prefix_cache = StrictRedis(host='localhost', port=6582, db=0, decode_responses=True)
self.running = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
self.storage = StrictRedis(host='localhost', port=16579, decode_responses=True)
self.intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
self.sanitize = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
self.prefix_cache = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
self.running = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
def get_runinng(self):
return self.running.hgetall('running')

View File

@ -6,7 +6,7 @@ from pathlib import Path
import logging
import json
import re
from redis import Redis
from redis import StrictRedis
from uuid import uuid4
from io import BytesIO
import importlib
@ -14,7 +14,7 @@ import importlib
from typing import List
import types
from .libs.helpers import safe_create_dir, set_running, unset_running
from .libs.helpers import safe_create_dir, set_running, unset_running, get_socket_path
class RawFilesParser():
@ -31,7 +31,7 @@ class RawFilesParser():
self.directory = storage_directory / self.vendor / self.listname
safe_create_dir(self.directory)
self.__init_logger(loglevel)
self.redis_intake = Redis(host='localhost', port=6579, db=0)
self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0)
self.logger.debug('Starting intake on {}'.format(self.source))
def __init_logger(self, loglevel):

View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import logging
from redis import Redis
from redis import StrictRedis
from ipaddress import ip_network
import requests
import gzip
@ -10,7 +10,7 @@ from io import BytesIO
from collections import defaultdict
import re
import time
from .libs.helpers import set_running, unset_running
from .libs.helpers import set_running, unset_running, get_socket_path
# Dataset source: Routeviews Prefix to AS mappings Dataset for IPv4 and IPv6
@ -21,7 +21,7 @@ class PrefixDatabase():
def __init__(self, loglevel: int=logging.DEBUG):
self.__init_logger(loglevel)
self.prefix_cache = Redis(host='localhost', port=6582, db=0, decode_responses=True)
self.prefix_cache = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
self.ipv6_url = 'http://data.caida.org/datasets/routing/routeviews6-prefix2as/{}'
self.ipv4_url = 'http://data.caida.org/datasets/routing/routeviews-prefix2as/{}'

View File

@ -2,12 +2,12 @@
# -*- coding: utf-8 -*-
import logging
from redis import Redis
from redis import StrictRedis
import time
import pytricia
import ipaddress
from .libs.helpers import shutdown_requested, set_running, unset_running
from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
class RISPrefixLookup():
@ -15,8 +15,8 @@ class RISPrefixLookup():
def __init__(self, loglevel: int=logging.DEBUG):
self.__init_logger(loglevel)
self.logger.info('Starting RIS Prefix fetcher')
self.prefix_db = Redis(host='localhost', port=6582, db=0, decode_responses=True)
self.longest_prefix_matching = Redis(host='localhost', port=6581, db=0, decode_responses=True)
self.prefix_db = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=0, decode_responses=True)
self.longest_prefix_matching = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
self.tree_v4 = pytricia.PyTricia()
self.tree_v6 = pytricia.PyTricia(128)
self.init_tree()

View File

@ -4,7 +4,7 @@
from dateutil import parser
import logging
from redis import StrictRedis
from .libs.helpers import shutdown_requested, set_running, unset_running
from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path
import ipaddress
@ -13,9 +13,9 @@ class Sanitizer():
def __init__(self, loglevel: int=logging.DEBUG):
self.__init_logger(loglevel)
self.redis_intake = StrictRedis(host='localhost', port=6579, db=0, decode_responses=True)
self.redis_sanitized = StrictRedis(host='localhost', port=6580, db=0, decode_responses=True)
self.ris_cache = StrictRedis(host='localhost', port=6581, db=0, decode_responses=True)
self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
self.ris_cache = StrictRedis(unix_socket_path=get_socket_path('ris'), db=0, decode_responses=True)
self.logger.debug('Starting import')
def __init_logger(self, loglevel):

View File

@ -1,12 +1,10 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bgpranking.libs.helpers import get_homedir
from bgpranking.libs.helpers import get_homedir, check_running
from subprocess import Popen
import time
from pathlib import Path
from redis import Redis
from redis.exceptions import ConnectionError
import argparse
@ -14,7 +12,7 @@ import argparse
def launch_cache(storage_directory: Path=None):
if not storage_directory:
storage_directory = get_homedir()
if not check_running('127.0.0.1', 6581) and not check_running('127.0.0.1', 6582):
if not check_running('ris') and not check_running('prefixes'):
Popen(["./run_redis.sh"], cwd=(storage_directory / 'cache'))
@ -27,7 +25,7 @@ def shutdown_cache(storage_directory: Path=None):
def launch_temp(storage_directory: Path=None):
if not storage_directory:
storage_directory = get_homedir()
if not check_running('127.0.0.1', 6579) and not check_running('127.0.0.1', 6580):
if not check_running('intake') and not check_running('prepare'):
Popen(["./run_redis.sh"], cwd=(storage_directory / 'temp'))
@ -40,7 +38,7 @@ def shutdown_temp(storage_directory: Path=None):
def launch_storage(storage_directory: Path=None):
if not storage_directory:
storage_directory = get_homedir()
if not check_running('127.0.0.1', 16579):
if not check_running('storage'):
Popen(["./run_ardb.sh"], cwd=(storage_directory / 'storage'))
@ -50,14 +48,6 @@ def shutdown_storage(storage_directory: Path=None):
Popen(["./shutdown_ardb.sh"], cwd=(storage_directory / 'storage'))
def check_running(host, port):
try:
r = Redis(host=host, port=port)
return r.ping()
except ConnectionError:
return False
def launch_all():
launch_cache()
launch_temp()
@ -65,26 +55,25 @@ def launch_all():
def check_all(stop=False):
backends = [['127.0.0.1', 6579, False], ['127.0.0.1', 6580, False],
['127.0.0.1', 6581, False], ['127.0.0.1', 6582, False],
['127.0.0.1', 16579, False]]
backends = [['ris', False], ['prefixes', False], ['storage', False],
['intake', False], ['prepare', False]]
while True:
for b in backends:
try:
b[2] = check_running(b[0], b[1])
b[1] = check_running(b[0])
except Exception:
b[2] = False
b[1] = False
if stop:
if not any(b[2] for b in backends):
if not any(b[1] for b in backends):
break
else:
if all(b[2] for b in backends):
if all(b[1] for b in backends):
break
for b in backends:
if not stop and not b[2]:
print('Waiting on {}:{}'.format(b[0], b[1]))
if stop and b[2]:
print('Waiting on {}:{}'.format(b[0], b[1]))
if not stop and not b[1]:
print('Waiting on {}'.format(b[0]))
if stop and b[1]:
print('Waiting on {}'.format(b[0]))
time.sleep(1)

View File

@ -1,12 +1,12 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bgpranking.libs.helpers import is_running
from bgpranking.libs.helpers import is_running, get_socket_path
import time
from redis import StrictRedis
if __name__ == '__main__':
r = StrictRedis(host='localhost', port=6582, db=1, decode_responses=True)
r = StrictRedis(unix_socket_path=get_socket_path('prefixes'), db=1, decode_responses=True)
r.set('shutdown', 1)
while True:
running = is_running()

View File

@ -89,7 +89,7 @@ protected-mode yes
# Accept connections on the specified port, default is 6379 (IANA #815344).
# If port 0 is specified Redis will not listen on a TCP socket.
port 6582
port 0
# TCP listen() backlog.
#
@ -106,8 +106,8 @@ tcp-backlog 511
# incoming connections. There is no default, so Redis will not listen
# on a unix socket when not specified.
#
# unixsocket /tmp/redis.sock
# unixsocketperm 700
unixsocket prefixes.sock
unixsocketperm 700
# Close the connection after a client is idle for N seconds (0 to disable)
timeout 0
@ -168,7 +168,7 @@ loglevel notice
# Specify the log file name. Also the empty string can be used to force
# Redis to log on the standard output. Note that if you use standard
# output for logging but daemonize, logs will be sent to /dev/null
logfile "ranking.log"
logfile "prefixes.log"
# To enable logging to the system logger, just set 'syslog-enabled' to yes,
# and optionally update the other syslog parameters to suit your needs.

View File

@ -89,7 +89,7 @@ protected-mode yes
# Accept connections on the specified port, default is 6379 (IANA #815344).
# If port 0 is specified Redis will not listen on a TCP socket.
port 6581
port 0
# TCP listen() backlog.
#
@ -106,8 +106,8 @@ tcp-backlog 511
# incoming connections. There is no default, so Redis will not listen
# on a unix socket when not specified.
#
# unixsocket /tmp/redis.sock
# unixsocketperm 700
unixsocket ris.sock
unixsocketperm 700
# Close the connection after a client is idle for N seconds (0 to disable)
timeout 0

4
cache/run_redis.sh vendored
View File

@ -3,5 +3,5 @@
set -e
set -x
../../redis/src/redis-server ./6581.conf
../../redis/src/redis-server ./6582.conf
../../redis/src/redis-server ./ris.conf
../../redis/src/redis-server ./prefixes.conf

View File

@ -3,5 +3,5 @@
# set -e
set -x
../../redis/src/redis-cli -p 6581 shutdown
../../redis/src/redis-cli -p 6582 shutdown
../../redis/src/redis-cli -s ./ris.sock shutdown
../../redis/src/redis-cli -s ./prefixes.sock shutdown

View File

@ -26,13 +26,13 @@ pidfile ${ARDB_HOME}/ardb.pid
thread-pool-size 4
#Accept connections on the specified host&port/unix socket, default is 0.0.0.0:16379.
server[0].listen 0.0.0.0:16579
#server[0].listen 0.0.0.0:16579
# If current qps exceed the limit, Ardb would return an error.
#server[0].qps-limit 1000
#listen on unix socket
#server[1].listen /tmp/ardb.sock
#server[1].unixsocketperm 755
server[0].listen storage.sock
server[0].unixsocketperm 755
#server[1].qps-limit 1000
# 'qps-limit-per-host' used to limit the request per second from same host

View File

@ -3,4 +3,4 @@
set -e
set -x
../../redis/src/redis-cli -p 16579 shutdown save
../../redis/src/redis-cli -s ./storage.sock shutdown save

View File

@ -89,7 +89,7 @@ protected-mode yes
# Accept connections on the specified port, default is 6379 (IANA #815344).
# If port 0 is specified Redis will not listen on a TCP socket.
port 6579
port 0
# TCP listen() backlog.
#
@ -106,8 +106,8 @@ tcp-backlog 511
# incoming connections. There is no default, so Redis will not listen
# on a unix socket when not specified.
#
# unixsocket /tmp/redis.sock
# unixsocketperm 700
unixsocket intake.sock
unixsocketperm 700
# Close the connection after a client is idle for N seconds (0 to disable)
timeout 0

View File

@ -89,7 +89,7 @@ protected-mode yes
# Accept connections on the specified port, default is 6379 (IANA #815344).
# If port 0 is specified Redis will not listen on a TCP socket.
port 6580
port 0
# TCP listen() backlog.
#
@ -106,8 +106,8 @@ tcp-backlog 511
# incoming connections. There is no default, so Redis will not listen
# on a unix socket when not specified.
#
# unixsocket /tmp/redis.sock
# unixsocketperm 700
unixsocket prepare.sock
unixsocketperm 700
# Close the connection after a client is idle for N seconds (0 to disable)
timeout 0
@ -250,7 +250,7 @@ rdbcompression yes
rdbchecksum yes
# The filename where to dump the DB
dbfilename pre_insert.rdb
dbfilename prepare.rdb
# The working directory.
#

View File

@ -3,5 +3,5 @@
set -e
set -x
../../redis/src/redis-server ./6579.conf
../../redis/src/redis-server ./6580.conf
../../redis/src/redis-server ./intake.conf
../../redis/src/redis-server ./prepare.conf

View File

@ -3,5 +3,5 @@
# set -e
set -x
../../redis/src/redis-cli -p 6579 shutdown
../../redis/src/redis-cli -p 6580 shutdown
../../redis/src/redis-cli -s ./intake.sock shutdown
../../redis/src/redis-cli -s ./prepare.sock shutdown