BGP-Ranking/bin/fetcher.py

44 lines
1.5 KiB
Python
Raw Normal View History

2018-03-29 22:37:28 +02:00
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import asyncio
from pathlib import Path
import aiohttp
2018-03-30 14:33:33 +02:00
from bgpranking.abstractmanager import AbstractManager
2018-03-29 22:37:28 +02:00
from bgpranking.modulesfetcher import Fetcher
from bgpranking.libs.helpers import get_config_path, get_homedir
2018-03-29 22:37:28 +02:00
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
level=logging.INFO, datefmt='%I:%M:%S')
2018-03-30 14:33:33 +02:00
class ModulesManager(AbstractManager):
2018-03-29 22:37:28 +02:00
def __init__(self, config_dir: Path=None, storage_directory: Path=None, loglevel: int=logging.DEBUG):
2018-03-30 14:33:33 +02:00
super().__init__(loglevel)
2018-03-29 22:37:28 +02:00
if not config_dir:
config_dir = get_config_path()
if not storage_directory:
storage_directory = get_homedir() / 'rawdata'
2018-03-29 22:37:28 +02:00
modules_config = config_dir / 'modules'
modules_paths = [modulepath for modulepath in modules_config.glob('*.json')]
self.modules = [Fetcher(path, storage_directory, loglevel) for path in modules_paths]
2018-03-30 14:33:33 +02:00
def _to_run_forever(self):
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(asyncio.gather(
*[module.fetch_list() for module in self.modules if module.fetcher])
)
except aiohttp.client_exceptions.ClientConnectorError as e:
self.logger.critical('Exception while fetching lists: {}'.format(e))
finally:
loop.close()
2018-03-29 22:37:28 +02:00
if __name__ == '__main__':
modules_manager = ModulesManager()
2018-03-30 14:33:33 +02:00
modules_manager.run(sleep_in_sec=3600)