CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of https://github.com/CIRCL/AIL-framework

pull/422/head
Terrtia 2019-05-24 13:27:44 +02:00
parent bc6332dae4 9c6a1ea8ec
commit 4b0a7210b8
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/Crawler.py
View File

@ -12,6 +12,7 @@ import time
import subprocess import subprocess
import requests import requests
from collections import deque
from pyfaup.faup import Faup from pyfaup.faup import Faup
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
@ -303,7 +304,7 @@ if __name__ == '__main__':
#mode = sys.argv[1] #mode = sys.argv[1]
splash_port = sys.argv[1] splash_port = sys.argv[1]
rotation_mode = ['onion', 'regular'] rotation_mode = deque(['onion', 'regular'])
default_proto_map = {'http': 80, 'https': 443} default_proto_map = {'http': 80, 'https': 443}
######################################################## add ftp ??? ######################################################## add ftp ???
@ -361,6 +362,7 @@ if __name__ == '__main__':
update_auto_crawler() update_auto_crawler()
rotation_mode.rotate()
to_crawl = get_elem_to_crawl(rotation_mode) to_crawl = get_elem_to_crawl(rotation_mode)
if to_crawl: if to_crawl:
url_data = unpack_url(to_crawl['url']) url_data = unpack_url(to_crawl['url'])

8
bin/packages/Paste.py
View File

@ -125,7 +125,15 @@ class Paste(object):
""" """
try:
paste = self.cache.get(self.p_path) paste = self.cache.get(self.p_path)
except UnicodeDecodeError:
paste = None
except Exception as e:
print("ERROR in: " + self.p_path)
print(e)
paste = None
if paste is None: if paste is None:
try: try:
with gzip.open(self.p_path, 'r') as f: with gzip.open(self.p_path, 'r') as f: