AIL-framework/bin/Onion.py

157 lines
5.7 KiB
Python
Raw Normal View History

#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
The ZMQ_Sub_Onion Module
============================
This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
2014-08-20 15:14:57 +02:00
It trying to extract url from paste and returning only ones which are tor
related (.onion)
..seealso:: Paste method (get_regex)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (Redis)
*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
"""
2014-08-14 17:55:18 +02:00
import pprint
import time
from packages import Paste
from pubsublogger import publisher
import datetime
import os
import base64
import subprocess
import redis
from Helper import Process
2014-08-14 17:55:18 +02:00
def fetch(p, r_cache, urls, domains, path):
2014-09-02 15:21:36 +02:00
failed = []
downloaded = []
print len(urls), 'Urls to fetch.'
for url, domain in zip(urls, domains):
2014-09-02 15:21:36 +02:00
if r_cache.exists(url) or url in failed:
continue
to_fetch = base64.standard_b64encode(url)
process = subprocess.Popen(["python", './tor_fetcher.py', to_fetch],
stdout=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
if process.returncode == 0:
r_cache.setbit(url, 0, 1)
r_cache.expire(url, 360000)
downloaded.append(url)
tempfile = process.stdout.read().strip()
with open(tempfile, 'r') as f:
filename = path + domain + '.gz'
fetched = f.read()
content = base64.standard_b64decode(fetched)
save_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"),
filename)
dirname = os.path.dirname(save_path)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(save_path, 'w') as ff:
ff.write(content)
p.populate_set_out(save_path, 'Global')
p.populate_set_out(url, 'ValidOnion')
p.populate_set_out(fetched, 'FetchedOnion')
yield url
os.unlink(tempfile)
else:
r_cache.setbit(url, 0, 0)
r_cache.expire(url, 3600)
2014-09-02 15:21:36 +02:00
failed.append(url)
print 'Failed at downloading', url
print process.stdout.read()
print 'Failed:', len(failed), 'Downloaded:', len(downloaded)
2014-08-20 15:14:57 +02:00
if __name__ == "__main__":
publisher.port = 6380
2014-08-20 15:14:57 +02:00
publisher.channel = "Script"
torclient_host = '127.0.0.1'
torclient_port = 9050
config_section = 'Onion'
p = Process(config_section)
r_cache = redis.StrictRedis(
host=p.config.get("Redis_Cache", "host"),
port=p.config.getint("Redis_Cache", "port"),
db=p.config.getint("Redis_Cache", "db"))
# FUNCTIONS #
publisher.info("Script subscribed to channel onion_categ")
# FIXME For retro compatibility
channel = 'onion_categ'
2014-08-14 17:55:18 +02:00
# Getting the first message from redis.
message = p.get_from_set()
prec_filename = None
2014-08-14 17:55:18 +02:00
# Thanks to Faup project for this regex
# https://github.com/stricaud/faup
url_regex = "((http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)"
while True:
2014-08-14 17:55:18 +02:00
if message is not None:
print message
2014-09-05 17:05:45 +02:00
filename, score = message.split()
# "For each new paste"
2014-08-14 17:55:18 +02:00
if prec_filename is None or filename != prec_filename:
domains_list = []
urls = []
2014-08-14 17:55:18 +02:00
PST = Paste.Paste(filename)
for x in PST.get_regex(url_regex):
2014-08-14 17:55:18 +02:00
# Extracting url with regex
url, s, credential, subdomain, domain, host, port, \
2014-08-20 15:14:57 +02:00
resource_path, query_string, f1, f2, f3, f4 = x
domains_list.append(domain)
urls.append(url)
2014-08-14 17:55:18 +02:00
# Saving the list of extracted onion domains.
PST.__setattr__(channel, domains_list)
2014-08-21 12:22:07 +02:00
PST.save_attribute_redis(channel, domains_list)
2014-08-20 15:14:57 +02:00
to_print = 'Onion;{};{};{};'.format(PST.p_source, PST.p_date,
PST.p_name)
if len(domains_list) > 0:
2014-08-20 15:14:57 +02:00
publisher.warning('{}Detected {} .onion(s)'.format(
to_print, len(domains_list)))
now = datetime.datetime.now()
path = os.path.join('onions', str(now.year).zfill(4),
str(now.month).zfill(2),
str(now.day).zfill(2),
str(int(time.mktime(now.utctimetuple()))))
to_print = 'Onion;{};{};{};'.format(PST.p_source,
PST.p_date,
PST.p_name)
for url in fetch(p, r_cache, urls, domains_list, path):
2014-09-02 15:21:36 +02:00
publisher.warning('{}Checked {}'.format(to_print, url))
else:
2014-08-14 17:55:18 +02:00
publisher.info('{}Onion related'.format(to_print))
prec_filename = filename
else:
publisher.debug("Script url is Idling 10s")
print 'Sleeping'
time.sleep(10)
message = p.get_from_set()