mirror of https://github.com/CIRCL/AIL-framework
Fix the exceptions
Raphaël Vinot
parent
9e8611a42d
commit
b7c9e489c9
55
bin/Mail.py
55
bin/Mail.py
|
|
@ -38,40 +38,35 @@ if __name__ == "__main__":
|
|||
is_critical = 10
|
||||
|
||||
email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}"
|
||||
|
||||
MX_values = None
|
||||
while True:
|
||||
try:
|
||||
if message is not None:
|
||||
print message
|
||||
filename, word, score = message.split()
|
||||
if message is not None:
|
||||
filename, word, score = message.split()
|
||||
|
||||
if prec_filename is None or filename != prec_filename:
|
||||
PST = Paste.Paste(filename)
|
||||
MX_values = lib_refine.checking_MX_record(
|
||||
r_serv2, PST.get_regex(email_regex))
|
||||
if prec_filename is None or filename != prec_filename:
|
||||
PST = Paste.Paste(filename)
|
||||
MX_values = lib_refine.checking_MX_record(
|
||||
r_serv2, PST.get_regex(email_regex))
|
||||
|
||||
if MX_values[0] >= 1:
|
||||
if MX_values[0] >= 1:
|
||||
|
||||
PST.__setattr__(channel, MX_values)
|
||||
PST.save_attribute_redis(channel, (MX_values[0],
|
||||
list(MX_values[1])))
|
||||
PST.__setattr__(channel, MX_values)
|
||||
PST.save_attribute_redis(channel, (MX_values[0],
|
||||
list(MX_values[1])))
|
||||
|
||||
pprint.pprint(MX_values)
|
||||
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
|
||||
format(PST.p_source, PST.p_date, PST.p_name,
|
||||
MX_values[0])
|
||||
if MX_values[0] > is_critical:
|
||||
publisher.warning(to_print)
|
||||
else:
|
||||
publisher.info(to_print)
|
||||
prec_filename = filename
|
||||
pprint.pprint(MX_values)
|
||||
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
|
||||
format(PST.p_source, PST.p_date, PST.p_name,
|
||||
MX_values[0])
|
||||
if MX_values[0] > is_critical:
|
||||
publisher.warning(to_print)
|
||||
else:
|
||||
publisher.info(to_print)
|
||||
prec_filename = filename
|
||||
|
||||
else:
|
||||
publisher.debug("Script Mails is Idling 10s")
|
||||
print 'Sleeping'
|
||||
time.sleep(10)
|
||||
else:
|
||||
publisher.debug("Script Mails is Idling 10s")
|
||||
print 'Sleeping'
|
||||
time.sleep(10)
|
||||
|
||||
message = p.get_from_set()
|
||||
except dns.exception.Timeout:
|
||||
# FIXME retry!
|
||||
print "dns.exception.Timeout"
|
||||
message = p.get_from_set()
|
||||
|
|
|
|||
115
bin/Url.py
115
bin/Url.py
|
|
@ -45,71 +45,68 @@ if __name__ == "__main__":
|
|||
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
||||
|
||||
while True:
|
||||
try:
|
||||
if message is not None:
|
||||
filename, word, score = message.split()
|
||||
if message is not None:
|
||||
filename, word, score = message.split()
|
||||
|
||||
if prec_filename is None or filename != prec_filename:
|
||||
domains_list = []
|
||||
PST = Paste.Paste(filename)
|
||||
client = ip2asn()
|
||||
for x in PST.get_regex(url_regex):
|
||||
scheme, credential, subdomain, domain, host, tld, \
|
||||
port, resource_path, query_string, f1, f2, f3, \
|
||||
f4 = x
|
||||
domains_list.append(domain)
|
||||
p.populate_set_out(x, 'Url')
|
||||
publisher.debug('{} Published'.format(x))
|
||||
if prec_filename is None or filename != prec_filename:
|
||||
domains_list = []
|
||||
PST = Paste.Paste(filename)
|
||||
client = ip2asn()
|
||||
for x in PST.get_regex(url_regex):
|
||||
scheme, credential, subdomain, domain, host, tld, \
|
||||
port, resource_path, query_string, f1, f2, f3, \
|
||||
f4 = x
|
||||
domains_list.append(domain)
|
||||
p.populate_set_out(x, 'Url')
|
||||
publisher.debug('{} Published'.format(x))
|
||||
|
||||
if f1 == "onion":
|
||||
print domain
|
||||
if f1 == "onion":
|
||||
print domain
|
||||
|
||||
hostl = unicode(subdomain+domain)
|
||||
try:
|
||||
socket.setdefaulttimeout(2)
|
||||
ip = socket.gethostbyname(unicode(hostl))
|
||||
except:
|
||||
# If the resolver is not giving any IPv4 address,
|
||||
# ASN/CC lookup is skip.
|
||||
continue
|
||||
hostl = unicode(subdomain+domain)
|
||||
try:
|
||||
socket.setdefaulttimeout(2)
|
||||
ip = socket.gethostbyname(unicode(hostl))
|
||||
except:
|
||||
# If the resolver is not giving any IPv4 address,
|
||||
# ASN/CC lookup is skip.
|
||||
continue
|
||||
|
||||
try:
|
||||
l = client.lookup(ip, qType='IP')
|
||||
except ipaddress.AddressValueError:
|
||||
continue
|
||||
cc = getattr(l, 'cc')
|
||||
asn = getattr(l, 'asn')
|
||||
try:
|
||||
l = client.lookup(ip, qType='IP')
|
||||
except ipaddress.AddressValueError:
|
||||
continue
|
||||
cc = getattr(l, 'cc')
|
||||
asn = getattr(l, 'asn')
|
||||
|
||||
# EU is not an official ISO 3166 code (but used by RIPE
|
||||
# IP allocation)
|
||||
if cc is not None and cc != "EU":
|
||||
print hostl, asn, cc, \
|
||||
pycountry.countries.get(alpha2=cc).name
|
||||
if cc == cc_critical:
|
||||
publisher.warning(
|
||||
'Url;{};{};{};Detected {} {}'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name,
|
||||
hostl, cc))
|
||||
else:
|
||||
print hostl, asn, cc
|
||||
# EU is not an official ISO 3166 code (but used by RIPE
|
||||
# IP allocation)
|
||||
if cc is not None and cc != "EU":
|
||||
print hostl, asn, cc, \
|
||||
pycountry.countries.get(alpha2=cc).name
|
||||
if cc == cc_critical:
|
||||
publisher.warning(
|
||||
'Url;{};{};{};Detected {} {}'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name,
|
||||
hostl, cc))
|
||||
else:
|
||||
print hostl, asn, cc
|
||||
|
||||
A_values = lib_refine.checking_A_record(r_serv2,
|
||||
domains_list)
|
||||
if A_values[0] >= 1:
|
||||
PST.__setattr__(channel, A_values)
|
||||
PST.save_attribute_redis(channel, (A_values[0],
|
||||
list(A_values[1])))
|
||||
A_values = lib_refine.checking_A_record(r_serv2,
|
||||
domains_list)
|
||||
if A_values[0] >= 1:
|
||||
PST.__setattr__(channel, A_values)
|
||||
PST.save_attribute_redis(channel, (A_values[0],
|
||||
list(A_values[1])))
|
||||
|
||||
pprint.pprint(A_values)
|
||||
publisher.info('Url;{};{};{};Checked {} URL'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
|
||||
prec_filename = filename
|
||||
pprint.pprint(A_values)
|
||||
publisher.info('Url;{};{};{};Checked {} URL'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
|
||||
prec_filename = filename
|
||||
|
||||
else:
|
||||
publisher.debug("Script url is Idling 10s")
|
||||
print 'Sleeping'
|
||||
time.sleep(10)
|
||||
else:
|
||||
publisher.debug("Script url is Idling 10s")
|
||||
print 'Sleeping'
|
||||
time.sleep(10)
|
||||
|
||||
message = p.get_from_set()
|
||||
except dns.exception.Timeout:
|
||||
print "dns.exception.Timeout", A_values
|
||||
message = p.get_from_set()
|
||||
|
|
|
|||
|
|
@ -33,7 +33,9 @@ def checking_MX_record(r_serv, adress_set):
|
|||
WalidMX = set([])
|
||||
# Transforming the set into a string
|
||||
MXdomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", str(adress_set).lower())
|
||||
|
||||
resolver = dns.resolver.Resolver()
|
||||
resolver.timeout = 5
|
||||
resolver.lifetime = 2
|
||||
if MXdomains != []:
|
||||
|
||||
for MXdomain in set(MXdomains):
|
||||
|
|
@ -45,9 +47,9 @@ def checking_MX_record(r_serv, adress_set):
|
|||
# Not already in Redis
|
||||
else:
|
||||
# If I'm Walid MX domain
|
||||
if dns.resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
|
||||
if resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
|
||||
# Gonna be added in redis.
|
||||
r_serv.setex(MXdomain[1:], timedelta(days=1), 1)
|
||||
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
|
||||
score += 1
|
||||
WalidMX.add(MXdomain[1:])
|
||||
else:
|
||||
|
|
@ -63,13 +65,17 @@ def checking_MX_record(r_serv, adress_set):
|
|||
publisher.debug('SyntaxError: EmptyLabel')
|
||||
|
||||
except dns.resolver.NXDOMAIN:
|
||||
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
|
||||
publisher.debug('The query name does not exist.')
|
||||
|
||||
except dns.name.LabelTooLong:
|
||||
publisher.debug('The Label is too long')
|
||||
|
||||
finally:
|
||||
pass
|
||||
except dns.resolver.Timeout:
|
||||
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
|
||||
|
||||
except Exception as e:
|
||||
print e
|
||||
|
||||
publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
|
||||
return (num, WalidMX)
|
||||
|
|
@ -79,6 +85,9 @@ def checking_A_record(r_serv, domains_set):
|
|||
score = 0
|
||||
num = len(domains_set)
|
||||
WalidA = set([])
|
||||
resolver = dns.resolver.Resolver()
|
||||
resolver.timeout = 5
|
||||
resolver.lifetime = 2
|
||||
|
||||
for Adomain in domains_set:
|
||||
try:
|
||||
|
|
@ -89,9 +98,9 @@ def checking_A_record(r_serv, domains_set):
|
|||
# Not already in Redis
|
||||
else:
|
||||
# If I'm Walid domain
|
||||
if dns.resolver.query(Adomain, rdtype=dns.rdatatype.A):
|
||||
if resolver.query(Adomain, rdtype=dns.rdatatype.A):
|
||||
# Gonna be added in redis.
|
||||
r_serv.setex(Adomain, timedelta(days=1), 1)
|
||||
r_serv.setex(Adomain, 1, timedelta(days=1))
|
||||
score += 1
|
||||
WalidA.add(Adomain)
|
||||
else:
|
||||
|
|
@ -107,13 +116,14 @@ def checking_A_record(r_serv, domains_set):
|
|||
publisher.debug('SyntaxError: EmptyLabel')
|
||||
|
||||
except dns.resolver.NXDOMAIN:
|
||||
r_serv.setex(Adomain[1:], 1, timedelta(days=1))
|
||||
publisher.debug('The query name does not exist.')
|
||||
|
||||
except dns.name.LabelTooLong:
|
||||
publisher.debug('The Label is too long')
|
||||
|
||||
finally:
|
||||
pass
|
||||
except Exception as e:
|
||||
print e
|
||||
|
||||
publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score))
|
||||
return (num, WalidA)
|
||||
|
|
|
|||
Loading…
Reference in New Issue