AIL-framework/bin/torcrawler/tor_crawler.py

50 lines
1.4 KiB
Python
Raw Normal View History

2018-08-09 17:42:21 +02:00
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
2019-02-25 16:38:50 +01:00
import json
import redis
2018-08-09 17:42:21 +02:00
from TorSplashCrawler import TorSplashCrawler
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import crawlers
2018-08-09 17:42:21 +02:00
if __name__ == '__main__':
2019-02-25 16:38:50 +01:00
if len(sys.argv) != 2:
print('usage:', 'tor_crawler.py', 'uuid')
2018-08-09 17:42:21 +02:00
exit(1)
config_loader = ConfigLoader.ConfigLoader()
redis_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None
2019-02-22 17:00:24 +01:00
2019-02-25 16:38:50 +01:00
# get crawler config key
uuid = sys.argv[1]
2019-02-22 17:00:24 +01:00
2019-02-25 16:38:50 +01:00
# get configs
crawler_json = json.loads(redis_cache.get('crawler_request:{}'.format(uuid)))
2019-02-22 17:00:24 +01:00
2019-02-25 16:38:50 +01:00
splash_url = crawler_json['splash_url']
service_type = crawler_json['service_type']
url = crawler_json['url']
domain = crawler_json['domain']
port = crawler_json['port']
2019-02-25 16:38:50 +01:00
original_item = crawler_json['item']
crawler_options = crawler_json['crawler_options']
date = crawler_json['date']
requested_mode = crawler_json['requested']
if crawler_options['cookiejar_uuid']:
cookies = crawlers.load_crawler_cookies(crawler_options['cookiejar_uuid'], domain, crawler_type=service_type)
else:
cookies = []
2019-02-22 17:00:24 +01:00
2019-02-25 16:38:50 +01:00
redis_cache.delete('crawler_request:{}'.format(uuid))
2019-02-21 09:54:43 +01:00
crawler = TorSplashCrawler(splash_url, crawler_options)
crawler.crawl(splash_url, service_type, crawler_options, date, requested_mode, url, domain, port, cookies, original_item)