mirror of https://github.com/CIRCL/lookyloo
fix: Rename scrape -> capture in async
parent
c7abc5df23
commit
ea052c7c12
|
@ -13,7 +13,7 @@ logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||||
level=logging.INFO, datefmt='%I:%M:%S')
|
level=logging.INFO, datefmt='%I:%M:%S')
|
||||||
|
|
||||||
|
|
||||||
class AsyncScraper(AbstractManager):
|
class AsyncCapture(AbstractManager):
|
||||||
|
|
||||||
def __init__(self, storage_directory: Optional[Path]=None, loglevel: int=logging.INFO):
|
def __init__(self, storage_directory: Optional[Path]=None, loglevel: int=logging.INFO):
|
||||||
super().__init__(loglevel)
|
super().__init__(loglevel)
|
||||||
|
@ -22,16 +22,16 @@ class AsyncScraper(AbstractManager):
|
||||||
self.lookyloo = Lookyloo()
|
self.lookyloo = Lookyloo()
|
||||||
|
|
||||||
def _to_run_forever(self):
|
def _to_run_forever(self):
|
||||||
set_running('async_scrape')
|
set_running('async_capture')
|
||||||
while True:
|
while True:
|
||||||
url = self.lookyloo.process_scrape_queue()
|
url = self.lookyloo.process_capture_queue()
|
||||||
if url is None or shutdown_requested():
|
if url is None or shutdown_requested():
|
||||||
break
|
break
|
||||||
unset_running('async_scrape')
|
unset_running('async_capture')
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
m = AsyncScraper()
|
m = AsyncCapture()
|
||||||
m.run(sleep_in_sec=1)
|
m.run(sleep_in_sec=1)
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@ def main():
|
||||||
p.check_returncode()
|
p.check_returncode()
|
||||||
print('done.')
|
print('done.')
|
||||||
print('Start asynchronous ingestor...')
|
print('Start asynchronous ingestor...')
|
||||||
Popen(['async_scrape'])
|
Popen(['async_capture'])
|
||||||
print('done.')
|
print('done.')
|
||||||
print('Start website...')
|
print('Start website...')
|
||||||
Popen(['start_website'])
|
Popen(['start_website'])
|
||||||
|
|
|
@ -23,7 +23,7 @@ classifiers = [
|
||||||
start_website = "bin.start_website:main"
|
start_website = "bin.start_website:main"
|
||||||
start = "bin.start:main"
|
start = "bin.start:main"
|
||||||
run_backend = "bin.run_backend:main"
|
run_backend = "bin.run_backend:main"
|
||||||
async_scrape = "bin.async_scrape:main"
|
async_capture = "bin.async_capture:main"
|
||||||
shutdown = "bin.shutdown:main"
|
shutdown = "bin.shutdown:main"
|
||||||
stop = "bin.stop:main"
|
stop = "bin.stop:main"
|
||||||
rebuild_caches = "bin.rebuild_caches:main"
|
rebuild_caches = "bin.rebuild_caches:main"
|
||||||
|
|
Loading…
Reference in New Issue