From b7c9e489c9c46852d9857fd588ea3833ceb5bf0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Sep 2014 11:46:07 +0200 Subject: [PATCH] Fix the exceptions --- bin/Mail.py | 55 ++++++++---------- bin/Url.py | 115 ++++++++++++++++++------------------- bin/packages/lib_refine.py | 28 ++++++--- 3 files changed, 100 insertions(+), 98 deletions(-) diff --git a/bin/Mail.py b/bin/Mail.py index 1632b7a0..d83c80e7 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -38,40 +38,35 @@ if __name__ == "__main__": is_critical = 10 email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}" - + MX_values = None while True: - try: - if message is not None: - print message - filename, word, score = message.split() + if message is not None: + filename, word, score = message.split() - if prec_filename is None or filename != prec_filename: - PST = Paste.Paste(filename) - MX_values = lib_refine.checking_MX_record( - r_serv2, PST.get_regex(email_regex)) + if prec_filename is None or filename != prec_filename: + PST = Paste.Paste(filename) + MX_values = lib_refine.checking_MX_record( + r_serv2, PST.get_regex(email_regex)) - if MX_values[0] >= 1: + if MX_values[0] >= 1: - PST.__setattr__(channel, MX_values) - PST.save_attribute_redis(channel, (MX_values[0], - list(MX_values[1]))) + PST.__setattr__(channel, MX_values) + PST.save_attribute_redis(channel, (MX_values[0], + list(MX_values[1]))) - pprint.pprint(MX_values) - to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\ - format(PST.p_source, PST.p_date, PST.p_name, - MX_values[0]) - if MX_values[0] > is_critical: - publisher.warning(to_print) - else: - publisher.info(to_print) - prec_filename = filename + pprint.pprint(MX_values) + to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\ + format(PST.p_source, PST.p_date, PST.p_name, + MX_values[0]) + if MX_values[0] > is_critical: + publisher.warning(to_print) + else: + publisher.info(to_print) + prec_filename = filename - else: - publisher.debug("Script Mails is Idling 10s") - print 'Sleeping' - time.sleep(10) + else: + publisher.debug("Script Mails is Idling 10s") + print 'Sleeping' + time.sleep(10) - message = p.get_from_set() - except dns.exception.Timeout: - # FIXME retry! - print "dns.exception.Timeout" + message = p.get_from_set() diff --git a/bin/Url.py b/bin/Url.py index e7a1bc8d..8ea2aead 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -45,71 +45,68 @@ if __name__ == "__main__": url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" while True: - try: - if message is not None: - filename, word, score = message.split() + if message is not None: + filename, word, score = message.split() - if prec_filename is None or filename != prec_filename: - domains_list = [] - PST = Paste.Paste(filename) - client = ip2asn() - for x in PST.get_regex(url_regex): - scheme, credential, subdomain, domain, host, tld, \ - port, resource_path, query_string, f1, f2, f3, \ - f4 = x - domains_list.append(domain) - p.populate_set_out(x, 'Url') - publisher.debug('{} Published'.format(x)) + if prec_filename is None or filename != prec_filename: + domains_list = [] + PST = Paste.Paste(filename) + client = ip2asn() + for x in PST.get_regex(url_regex): + scheme, credential, subdomain, domain, host, tld, \ + port, resource_path, query_string, f1, f2, f3, \ + f4 = x + domains_list.append(domain) + p.populate_set_out(x, 'Url') + publisher.debug('{} Published'.format(x)) - if f1 == "onion": - print domain + if f1 == "onion": + print domain - hostl = unicode(subdomain+domain) - try: - socket.setdefaulttimeout(2) - ip = socket.gethostbyname(unicode(hostl)) - except: - # If the resolver is not giving any IPv4 address, - # ASN/CC lookup is skip. - continue + hostl = unicode(subdomain+domain) + try: + socket.setdefaulttimeout(2) + ip = socket.gethostbyname(unicode(hostl)) + except: + # If the resolver is not giving any IPv4 address, + # ASN/CC lookup is skip. + continue - try: - l = client.lookup(ip, qType='IP') - except ipaddress.AddressValueError: - continue - cc = getattr(l, 'cc') - asn = getattr(l, 'asn') + try: + l = client.lookup(ip, qType='IP') + except ipaddress.AddressValueError: + continue + cc = getattr(l, 'cc') + asn = getattr(l, 'asn') - # EU is not an official ISO 3166 code (but used by RIPE - # IP allocation) - if cc is not None and cc != "EU": - print hostl, asn, cc, \ - pycountry.countries.get(alpha2=cc).name - if cc == cc_critical: - publisher.warning( - 'Url;{};{};{};Detected {} {}'.format( - PST.p_source, PST.p_date, PST.p_name, - hostl, cc)) - else: - print hostl, asn, cc + # EU is not an official ISO 3166 code (but used by RIPE + # IP allocation) + if cc is not None and cc != "EU": + print hostl, asn, cc, \ + pycountry.countries.get(alpha2=cc).name + if cc == cc_critical: + publisher.warning( + 'Url;{};{};{};Detected {} {}'.format( + PST.p_source, PST.p_date, PST.p_name, + hostl, cc)) + else: + print hostl, asn, cc - A_values = lib_refine.checking_A_record(r_serv2, - domains_list) - if A_values[0] >= 1: - PST.__setattr__(channel, A_values) - PST.save_attribute_redis(channel, (A_values[0], - list(A_values[1]))) + A_values = lib_refine.checking_A_record(r_serv2, + domains_list) + if A_values[0] >= 1: + PST.__setattr__(channel, A_values) + PST.save_attribute_redis(channel, (A_values[0], + list(A_values[1]))) - pprint.pprint(A_values) - publisher.info('Url;{};{};{};Checked {} URL'.format( - PST.p_source, PST.p_date, PST.p_name, A_values[0])) - prec_filename = filename + pprint.pprint(A_values) + publisher.info('Url;{};{};{};Checked {} URL'.format( + PST.p_source, PST.p_date, PST.p_name, A_values[0])) + prec_filename = filename - else: - publisher.debug("Script url is Idling 10s") - print 'Sleeping' - time.sleep(10) + else: + publisher.debug("Script url is Idling 10s") + print 'Sleeping' + time.sleep(10) - message = p.get_from_set() - except dns.exception.Timeout: - print "dns.exception.Timeout", A_values + message = p.get_from_set() diff --git a/bin/packages/lib_refine.py b/bin/packages/lib_refine.py index 45b1d009..43d3654b 100644 --- a/bin/packages/lib_refine.py +++ b/bin/packages/lib_refine.py @@ -33,7 +33,9 @@ def checking_MX_record(r_serv, adress_set): WalidMX = set([]) # Transforming the set into a string MXdomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", str(adress_set).lower()) - + resolver = dns.resolver.Resolver() + resolver.timeout = 5 + resolver.lifetime = 2 if MXdomains != []: for MXdomain in set(MXdomains): @@ -45,9 +47,9 @@ def checking_MX_record(r_serv, adress_set): # Not already in Redis else: # If I'm Walid MX domain - if dns.resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX): + if resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX): # Gonna be added in redis. - r_serv.setex(MXdomain[1:], timedelta(days=1), 1) + r_serv.setex(MXdomain[1:], 1, timedelta(days=1)) score += 1 WalidMX.add(MXdomain[1:]) else: @@ -63,13 +65,17 @@ def checking_MX_record(r_serv, adress_set): publisher.debug('SyntaxError: EmptyLabel') except dns.resolver.NXDOMAIN: + r_serv.setex(MXdomain[1:], 1, timedelta(days=1)) publisher.debug('The query name does not exist.') except dns.name.LabelTooLong: publisher.debug('The Label is too long') - finally: - pass + except dns.resolver.Timeout: + r_serv.setex(MXdomain[1:], 1, timedelta(days=1)) + + except Exception as e: + print e publisher.debug("emails before: {0} after: {1} (valid)".format(num, score)) return (num, WalidMX) @@ -79,6 +85,9 @@ def checking_A_record(r_serv, domains_set): score = 0 num = len(domains_set) WalidA = set([]) + resolver = dns.resolver.Resolver() + resolver.timeout = 5 + resolver.lifetime = 2 for Adomain in domains_set: try: @@ -89,9 +98,9 @@ def checking_A_record(r_serv, domains_set): # Not already in Redis else: # If I'm Walid domain - if dns.resolver.query(Adomain, rdtype=dns.rdatatype.A): + if resolver.query(Adomain, rdtype=dns.rdatatype.A): # Gonna be added in redis. - r_serv.setex(Adomain, timedelta(days=1), 1) + r_serv.setex(Adomain, 1, timedelta(days=1)) score += 1 WalidA.add(Adomain) else: @@ -107,13 +116,14 @@ def checking_A_record(r_serv, domains_set): publisher.debug('SyntaxError: EmptyLabel') except dns.resolver.NXDOMAIN: + r_serv.setex(Adomain[1:], 1, timedelta(days=1)) publisher.debug('The query name does not exist.') except dns.name.LabelTooLong: publisher.debug('The Label is too long') - finally: - pass + except Exception as e: + print e publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score)) return (num, WalidA)