D4
/
BGP-Ranking
mirror of https://github.com/D4-project/BGP-Ranking
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Force usage of redis-py 3+

pull/12/head
Raphaël Vinot 2018-11-26 14:59:53 +01:00
parent aae34e324d
commit 367f2f8421
3 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bgpranking/parser.py
View File

@ -11,7 +11,7 @@ from uuid import uuid4
from io import BytesIO from io import BytesIO
import importlib import importlib
from typing import List, Union, Tuple from typing import List
import types import types
from .libs.helpers import safe_create_dir, set_running, unset_running, get_socket_path from .libs.helpers import safe_create_dir, set_running, unset_running, get_socket_path
@ -74,9 +74,9 @@ class RawFilesParser():
else: else:
datetime = self.datetime datetime = self.datetime
uuid = uuid4() uuid = uuid4()
p.hmset(uuid, {'ip': ip, 'source': self.source, p.hmset(str(uuid), {'ip': ip, 'source': self.source,
'datetime': datetime.isoformat()}) 'datetime': datetime.isoformat()})
p.sadd('intake', uuid) p.sadd('intake', str(uuid))
p.execute() p.execute()
self._archive(filepath) self._archive(filepath)
except Exception as e: except Exception as e:

14
bgpranking/ranking.py
View File

@ -51,13 +51,13 @@ class Ranking():
for ip_ts in self.storage.smembers(f'{day}|{source}|{asn}|{prefix}')]) for ip_ts in self.storage.smembers(f'{day}|{source}|{asn}|{prefix}')])
py_prefix = ip_network(prefix) py_prefix = ip_network(prefix)
prefix_rank = float(len(ips)) / py_prefix.num_addresses prefix_rank = float(len(ips)) / py_prefix.num_addresses
r_pipeline.zadd(f'{day}|{source}|{asn}|v{py_prefix.version}|prefixes', prefix_rank, prefix) r_pipeline.zadd(f'{day}|{source}|{asn}|v{py_prefix.version}|prefixes', {prefix: prefix_rank})
if py_prefix.version == 4: if py_prefix.version == 4:
asn_rank_v4 += len(ips) * self.config_files[source]['impact'] asn_rank_v4 += len(ips) * self.config_files[source]['impact']
r_pipeline.zincrby(prefixes_aggregation_key_v4, prefix, prefix_rank * self.config_files[source]['impact']) r_pipeline.zincrby(prefixes_aggregation_key_v4, prefix_rank * self.config_files[source]['impact'], prefix)
else: else:
asn_rank_v6 += len(ips) * self.config_files[source]['impact'] asn_rank_v6 += len(ips) * self.config_files[source]['impact']
r_pipeline.zincrby(prefixes_aggregation_key_v6, prefix, prefix_rank * self.config_files[source]['impact']) r_pipeline.zincrby(prefixes_aggregation_key_v6, prefix_rank * self.config_files[source]['impact'], prefix)
v4info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v4', date=day) v4info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v4', date=day)
v6info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v6', date=day) v6info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v6', date=day)
ipasnhistory_date_v4 = list(v4info['response'].keys())[0] ipasnhistory_date_v4 = list(v4info['response'].keys())[0]
@ -68,14 +68,14 @@ class Ranking():
asn_rank_v4 /= float(v4count) asn_rank_v4 /= float(v4count)
if asn_rank_v4: if asn_rank_v4:
r_pipeline.set(f'{day}|{source}|{asn}|v4', asn_rank_v4) r_pipeline.set(f'{day}|{source}|{asn}|v4', asn_rank_v4)
r_pipeline.zincrby(asns_aggregation_key_v4, asn, asn_rank_v4) r_pipeline.zincrby(asns_aggregation_key_v4, asn_rank_v4, asn)
r_pipeline.zadd(source_aggregation_key_v4, asn_rank_v4, asn) r_pipeline.zadd(source_aggregation_key_v4, {asn: asn_rank_v4})
if v6count: if v6count:
asn_rank_v6 /= float(v6count) asn_rank_v6 /= float(v6count)
if asn_rank_v6: if asn_rank_v6:
r_pipeline.set(f'{day}|{source}|{asn}|v6', asn_rank_v6) r_pipeline.set(f'{day}|{source}|{asn}|v6', asn_rank_v6)
r_pipeline.zincrby(asns_aggregation_key_v6, asn, asn_rank_v6) r_pipeline.zincrby(asns_aggregation_key_v6, asn_rank_v6, asn)
r_pipeline.zadd(source_aggregation_key_v6, asn_rank_v4, asn) r_pipeline.zadd(source_aggregation_key_v6, {asn: asn_rank_v6})
self.ranking.delete(*to_delete) self.ranking.delete(*to_delete)
r_pipeline.execute() r_pipeline.execute()

6
bin/start.py
View File

@ -4,6 +4,12 @@
from subprocess import Popen from subprocess import Popen
from bgpranking.libs.helpers import get_homedir from bgpranking.libs.helpers import get_homedir
import redis
import sys
if redis.VERSION < (3, ):
print('redis-py >= 3 is required.')
sys.exit()
if __name__ == '__main__': if __name__ == '__main__':
# Just fail if the env isn't set. # Just fail if the env isn't set.