From 9b66fa703fe46403960b83a70bfa6df13698ec63 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 3 May 2019 10:04:20 +0200 Subject: [PATCH 01/33] fix: [crawler] typo: domains down --- var/www/modules/hiddenServices/Flask_hiddenServices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index fd68dc93..2d45b9c6 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -794,7 +794,7 @@ def domain_crawled_by_type_json(): day_crawled = {} day_crawled['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] day_crawled['UP']= nb_domain_up = r_serv_onion.scard('{}_up:{}'.format(type, date)) - day_crawled['DOWN']= nb_domain_up = r_serv_onion.scard('{}_up:{}'.format(type, date)) + day_crawled['DOWN']= nb_domain_up = r_serv_onion.scard('{}_down:{}'.format(type, date)) range_decoder.append(day_crawled) return jsonify(range_decoder) From 97fa1ddca4c201f43eb6d3c986f96e47fd383d64 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 3 May 2019 17:02:56 +0200 Subject: [PATCH 02/33] fix: [python requirements] rename file --- ansible/roles/ail-host/tasks/main.yml | 8 +------- installing_deps.sh | 2 +- pip3_packages_requirement.txt => requirements.txt | 0 3 files changed, 2 insertions(+), 8 deletions(-) rename pip3_packages_requirement.txt => requirements.txt (100%) diff --git a/ansible/roles/ail-host/tasks/main.yml b/ansible/roles/ail-host/tasks/main.yml index 43b23a8e..af520fe4 100644 --- a/ansible/roles/ail-host/tasks/main.yml +++ b/ansible/roles/ail-host/tasks/main.yml @@ -145,15 +145,9 @@ extra_args: --upgrade virtualenv: /opt/AIL-framework/AILENV -- name: Install pip requirements - pip: - requirements: /opt/AIL-framework/pip_packages_requirement.txt - extra_args: --upgrade - virtualenv: /opt/AIL-framework/AILENV - - name: Install pip3 requirements pip: - requirements: /opt/AIL-framework/pip3_packages_requirement.txt + requirements: /opt/AIL-framework/requirements.txt executable: pip3 extra_args: --upgrade diff --git a/installing_deps.sh b/installing_deps.sh index f8f2846f..f3e1d7b9 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -99,7 +99,7 @@ mkdir -p $AIL_HOME/PASTES pip3 install -U pip pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client' -pip3 install -U -r pip3_packages_requirement.txt +pip3 install -U -r requirements.txt # Pyfaup pushd faup/src/lib/bindings/python/ diff --git a/pip3_packages_requirement.txt b/requirements.txt similarity index 100% rename from pip3_packages_requirement.txt rename to requirements.txt From cc61c99290900b164a11da49de608d2adc3c8286 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 6 May 2019 10:00:39 +0200 Subject: [PATCH 03/33] fix: [showpaste] fix: #346, avoid None screenshots --- var/www/modules/showpaste/Flask_showpaste.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index 474280b5..b94a1073 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -44,6 +44,8 @@ def get_item_screenshot_path(item): screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot') if screenshot: screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:]) + else: + screenshot = '' return screenshot def showpaste(content_range, requested_path): From a4c03b4ba4f24cd53bb700836bf8000d92b0c90f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 6 May 2019 11:46:20 +0200 Subject: [PATCH 04/33] fix: [Crawler] force domains/subdomains lower case (rfc4343) --- bin/Crawler.py | 48 ++++++++++++++++++++++++++++++++++++++---------- bin/Onion.py | 8 ++++++-- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/bin/Crawler.py b/bin/Crawler.py index a7e9365b..d8e6a430 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -43,25 +43,49 @@ def unpack_url(url): to_crawl = {} faup.decode(url) url_unpack = faup.get() - to_crawl['domain'] = url_unpack['domain'].decode() + # # FIXME: # TODO: remove me + try: + to_crawl['domain'] = url_unpack['domain'].decode() + except: + to_crawl['domain'] = url_unpack['domain'] + to_crawl['domain'] = to_crawl['domain'].lower() + + + # force lower case domain/subdomain (rfc4343) + # # FIXME: # TODO: remove me + try: + url_host = url_unpack['host'].decode() + except: + url_host = url_unpack['host'] + + new_url_host = url_host.lower() + url_lower_case = url.replace(url_host, new_url_host, 1) if url_unpack['scheme'] is None: to_crawl['scheme'] = 'http' - url= 'http://{}'.format(url_unpack['url'].decode()) + url= 'http://{}'.format(url_lower_case) else: - scheme = url_unpack['scheme'].decode() + # # FIXME: # TODO: remove me + try: + scheme = url_unpack['scheme'].decode() + except Exception as e: + scheme = url_unpack['scheme'] if scheme in default_proto_map: to_crawl['scheme'] = scheme - url = url_unpack['url'].decode() + url = url_lower_case else: - redis_crawler.sadd('new_proto', '{} {}'.format(scheme, url_unpack['url'].decode())) + redis_crawler.sadd('new_proto', '{} {}'.format(scheme, url_lower_case)) to_crawl['scheme'] = 'http' - url= 'http://{}'.format(url_unpack['url'].decode().replace(scheme, '', 1)) + url= 'http://{}'.format(url_lower_case.replace(scheme, '', 1)) if url_unpack['port'] is None: to_crawl['port'] = default_proto_map[to_crawl['scheme']] else: - port = url_unpack['port'].decode() + # # FIXME: # TODO: remove me + try: + port = url_unpack['port'].decode() + except: + port = url_unpack['port'] # Verify port number #################### make function to verify/correct port number try: int(port) @@ -80,12 +104,16 @@ def unpack_url(url): to_crawl['url'] = url if to_crawl['port'] == 80: - to_crawl['domain_url'] = '{}://{}'.format(to_crawl['scheme'], url_unpack['host'].decode()) + to_crawl['domain_url'] = '{}://{}'.format(to_crawl['scheme'], new_url_host) else: - to_crawl['domain_url'] = '{}://{}:{}'.format(to_crawl['scheme'], url_unpack['host'].decode(), to_crawl['port']) + to_crawl['domain_url'] = '{}://{}:{}'.format(to_crawl['scheme'], new_url_host, to_crawl['port']) + # # FIXME: # TODO: remove me + try: + to_crawl['tld'] = url_unpack['tld'].decode() + except: + to_crawl['tld'] = url_unpack['tld'] - to_crawl['tld'] = url_unpack['tld'].decode() return to_crawl # get url, paste and service_type to crawl diff --git a/bin/Onion.py b/bin/Onion.py index cbe8bb9f..2aa56d8e 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -224,7 +224,11 @@ if __name__ == "__main__": faup.decode(url) url_unpack = faup.get() - domain = url_unpack['domain'].decode() + ## TODO: # FIXME: remove me + try: + domain = url_unpack['domain'].decode().lower() + except Exception as e: + domain = url_unpack['domain'].lower() ## TODO: blackilst by port ? # check blacklist @@ -233,7 +237,7 @@ if __name__ == "__main__": subdomain = re.findall(url_regex, url) if len(subdomain) > 0: - subdomain = subdomain[0][4] + subdomain = subdomain[0][4].lower() else: continue From 2606220c2bbc74a6c351c8713f84d6d4a111b02e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 6 May 2019 13:38:13 +0200 Subject: [PATCH 05/33] fix: [faup] fix new return types (bytes to str) --- bin/Credential.py | 10 ++++++ bin/LibInjection.py | 19 ++++++++-- bin/Mail.py | 4 +++ bin/SQLInjectionDetection.py | 19 ++++++++-- bin/Web.py | 16 +++++---- .../hiddenServices/Flask_hiddenServices.py | 35 ++++++++++++++++--- 6 files changed, 87 insertions(+), 16 deletions(-) diff --git a/bin/Credential.py b/bin/Credential.py index 8da84883..2eeb3d55 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -124,6 +124,11 @@ if __name__ == "__main__": for url in sites: faup.decode(url) domain = faup.get()['domain'] + ## TODO: # FIXME: remove me + try: + domain = domain.decode() + except: + pass if domain in creds_sites.keys(): creds_sites[domain] += 1 else: @@ -143,6 +148,11 @@ if __name__ == "__main__": maildomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", cred.lower())[0] faup.decode(maildomains) tld = faup.get()['tld'] + ## TODO: # FIXME: remove me + try: + tld = tld.decode() + except: + pass server_statistics.hincrby('credential_by_tld:'+date, tld, 1) else: publisher.info(to_print) diff --git a/bin/LibInjection.py b/bin/LibInjection.py index 4b851f21..65789e62 100755 --- a/bin/LibInjection.py +++ b/bin/LibInjection.py @@ -29,8 +29,17 @@ def analyse(url, path): faup.decode(url) url_parsed = faup.get() pprint.pprint(url_parsed) - resource_path = url_parsed['resource_path'] - query_string = url_parsed['query_string'] + ## TODO: # FIXME: remove me + try: + resource_path = url_parsed['resource_path'].encode() + except: + resource_path = url_parsed['resource_path'] + + ## TODO: # FIXME: remove me + try: + query_string = url_parsed['query_string'].encode() + except: + query_string = url_parsed['query_string'] result_path = {'sqli' : False} result_query = {'sqli' : False} @@ -56,7 +65,11 @@ def analyse(url, path): p.populate_set_out(msg, 'Tags') #statistics - tld = url_parsed['tld'] + ## TODO: # FIXME: remove me + try: + tld = url_parsed['tld'].decode() + except: + tld = url_parsed['tld'] if tld is not None: date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1) diff --git a/bin/Mail.py b/bin/Mail.py index b8311d3c..b90ee4ab 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -95,6 +95,10 @@ if __name__ == "__main__": faup.decode(mail) tld = faup.get()['tld'] + try: + tld = tld.decode() + except: + pass server_statistics.hincrby('mail_by_tld:'+date, tld, MX_values[1][mail]) else: diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index a5595dc7..ffb21003 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -68,10 +68,20 @@ def analyse(url, path): result_query = 0 if resource_path is not None: - result_path = is_sql_injection(resource_path.decode('utf8')) + ## TODO: # FIXME: remove me + try: + resource_path = resource_path.decode() + except: + pass + result_path = is_sql_injection(resource_path) if query_string is not None: - result_query = is_sql_injection(query_string.decode('utf8')) + ## TODO: # FIXME: remove me + try: + query_string = query_string.decode() + except: + pass + result_query = is_sql_injection(query_string) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) @@ -89,6 +99,11 @@ def analyse(url, path): #statistics tld = url_parsed['tld'] if tld is not None: + ## TODO: # FIXME: remove me + try: + tld = tld.decode() + except: + pass date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1) diff --git a/bin/Web.py b/bin/Web.py index 7cc96822..ca4366e8 100755 --- a/bin/Web.py +++ b/bin/Web.py @@ -94,18 +94,22 @@ if __name__ == "__main__": faup.decode(url) domain = faup.get_domain() subdomain = faup.get_subdomain() - f1 = None publisher.debug('{} Published'.format(url)) - if f1 == "onion": - print(domain) - if subdomain is not None: - subdomain = subdomain.decode('utf8') + ## TODO: # FIXME: remove me + try: + subdomain = subdomain.decode() + except: + pass if domain is not None: - domain = domain.decode('utf8') + ## TODO: # FIXME: remove me + try: + domain = domain.decode() + except: + pass domains_list.append(domain) hostl = avoidNone(subdomain) + avoidNone(domain) diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 2d45b9c6..0ee44974 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -115,7 +115,12 @@ def get_type_domain(domain): def get_domain_from_url(url): faup.decode(url) unpack_url = faup.get() - domain = unpack_url['domain'].decode() + domain = unpack_url['domain'] + ## TODO: FIXME remove me + try: + domain = domain.decode() + except: + pass return domain def get_last_domains_crawled(type): @@ -418,8 +423,19 @@ def create_spider_splash(): # get service_type faup.decode(url) unpack_url = faup.get() - domain = unpack_url['domain'].decode() - if unpack_url['tld'] == b'onion': + ## TODO: # FIXME: remove me + try: + domain = unpack_url['domain'].decode() + except: + domain = unpack_url['domain'] + + ## TODO: # FIXME: remove me + try: + tld = unpack_url['tld'].decode() + except: + tld = unpack_url['tld'] + + if tld == 'onion': service_type = 'onion' else: service_type = 'regular' @@ -694,10 +710,19 @@ def show_domain(): port = request.args.get('port') faup.decode(domain) unpack_url = faup.get() - domain = unpack_url['domain'].decode() + + ## TODO: # FIXME: remove me + try: + domain = unpack_url['domain'].decode() + except: + domain = unpack_url['domain'] + if not port: if unpack_url['port']: - port = unpack_url['port'].decode() + try: + port = unpack_url['port'].decode() + except: + port = unpack_url['port'] else: port = 80 try: From 254441f19341da351adb7538d11f74aeee1120bc Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 13 May 2019 13:56:43 +0200 Subject: [PATCH 06/33] chg: [crawler] manual/auto crawler: always save screenshots --- bin/torcrawler/TorSplashCrawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/torcrawler/TorSplashCrawler.py b/bin/torcrawler/TorSplashCrawler.py index d1417e6e..17710bf6 100644 --- a/bin/torcrawler/TorSplashCrawler.py +++ b/bin/torcrawler/TorSplashCrawler.py @@ -57,6 +57,7 @@ class TorSplashCrawler(): def __init__(self, type, crawler_options, date, url, domain, port, original_item, *args, **kwargs): self.type = type + self.requested = crawler_options['requested'] self.original_item = original_item self.root_key = None self.start_urls = url @@ -183,7 +184,7 @@ class TorSplashCrawler(): if 'png' in response.data: size_screenshot = (len(response.data['png'])*3) /4 - if size_screenshot < 5000000: #bytes + if size_screenshot < 5000000 or self.requested: #bytes or manual/auto image_content = base64.standard_b64decode(response.data['png'].encode()) hash = sha256(image_content).hexdigest() img_dir_path = os.path.join(hash[0:2], hash[2:4], hash[4:6], hash[6:8], hash[8:10], hash[10:12]) From 0389b9c23b9280c74fc6b66911672415d4d944d0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 13 May 2019 14:24:16 +0200 Subject: [PATCH 07/33] chg: [crawler] manual/auto crawler: always save screenshots --- bin/torcrawler/TorSplashCrawler.py | 10 +++++----- bin/torcrawler/tor_crawler.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/torcrawler/TorSplashCrawler.py b/bin/torcrawler/TorSplashCrawler.py index 17710bf6..4e36c1c9 100644 --- a/bin/torcrawler/TorSplashCrawler.py +++ b/bin/torcrawler/TorSplashCrawler.py @@ -48,16 +48,16 @@ class TorSplashCrawler(): 'DEPTH_LIMIT': crawler_options['depth_limit'] }) - def crawl(self, type, crawler_options, date, url, domain, port, original_item): - self.process.crawl(self.crawler, type=type, crawler_options=crawler_options, date=date, url=url, domain=domain, port=port, original_item=original_item) + def crawl(self, type, crawler_options, date, requested_mode, url, domain, port, original_item): + self.process.crawl(self.crawler, type=type, crawler_options=crawler_options, date=date, requested_mode=requested_mode, url=url, domain=domain, port=port, original_item=original_item) self.process.start() class TorSplashSpider(Spider): name = 'TorSplashSpider' - def __init__(self, type, crawler_options, date, url, domain, port, original_item, *args, **kwargs): + def __init__(self, type, crawler_options, date, requested_mode, url, domain, port, original_item, *args, **kwargs): self.type = type - self.requested = crawler_options['requested'] + self.requested_mode = requested_mode self.original_item = original_item self.root_key = None self.start_urls = url @@ -184,7 +184,7 @@ class TorSplashCrawler(): if 'png' in response.data: size_screenshot = (len(response.data['png'])*3) /4 - if size_screenshot < 5000000 or self.requested: #bytes or manual/auto + if size_screenshot < 5000000 or self.requested_mode: #bytes or manual/auto image_content = base64.standard_b64decode(response.data['png'].encode()) hash = sha256(image_content).hexdigest() img_dir_path = os.path.join(hash[0:2], hash[2:4], hash[4:6], hash[6:8], hash[8:10], hash[10:12]) diff --git a/bin/torcrawler/tor_crawler.py b/bin/torcrawler/tor_crawler.py index 13a67545..2d8365c4 100755 --- a/bin/torcrawler/tor_crawler.py +++ b/bin/torcrawler/tor_crawler.py @@ -43,8 +43,9 @@ if __name__ == '__main__': original_item = crawler_json['item'] crawler_options = crawler_json['crawler_options'] date = crawler_json['date'] + requested_mode = crawler_json['requested'] redis_cache.delete('crawler_request:{}'.format(uuid)) crawler = TorSplashCrawler(splash_url, crawler_options) - crawler.crawl(service_type, crawler_options, date, url, domain, port, original_item) + crawler.crawl(service_type, crawler_options, date, requested_mode, url, domain, port, original_item) From 05079c143c562b74abe1e95e018bbe9e7cdc7d89 Mon Sep 17 00:00:00 2001 From: kovacsbalu Date: Tue, 14 May 2019 16:06:20 +0200 Subject: [PATCH 08/33] Fix #314 Replace char on redis encoding error. Try to use local file on other error. --- bin/packages/Paste.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index f8be2f9b..884bd5ee 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -70,6 +70,7 @@ class Paste(object): host=cfg.get("Redis_Queues", "host"), port=cfg.getint("Redis_Queues", "port"), db=cfg.getint("Redis_Queues", "db"), + encoding_errors='replace', decode_responses=True) self.store = redis.StrictRedis( host=cfg.get("Redis_Data_Merging", "host"), @@ -125,7 +126,13 @@ class Paste(object): """ - paste = self.cache.get(self.p_path) + try: + paste = self.cache.get(self.p_path) + except Exception as e: + print("ERROR in: " + self.p_path) + print(e) + paste = None + if paste is None: try: with gzip.open(self.p_path, 'r') as f: From 648074464170a9899b7815b162e9eee7400a0be4 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 14 May 2019 17:49:31 +0200 Subject: [PATCH 09/33] chg: [PgpDump] add PgpDump backend TODO: UI --- OVERVIEW.md | 50 +++++++++ bin/Keys.py | 9 ++ bin/PgpDump.py | 188 +++++++++++++++++++++++++++++++++ bin/packages/config.cfg.sample | 3 + bin/packages/modules.cfg | 4 + 5 files changed, 254 insertions(+) create mode 100755 bin/PgpDump.py diff --git a/OVERVIEW.md b/OVERVIEW.md index 5d6dbc99..2b357f9f 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -152,6 +152,56 @@ Redis and ARDB overview | binary_hash:**hash** | **item** | **nb_seen** | | hexadecimal_hash:**hash** | **item** | **nb_seen** | +#### PgpDump + +##### Hset: +| Key | Field | Value | +| ------ | ------ | ------ | +| pgp_metadata_key:*key id* | first_seen | **date** | +| | last_seen | **date** | +| | | +| pgp_metadata_name:*name* | first_seen | **date** | +| | last_seen | **date** | +| | | +| pgp_metadata_mail:*mail* | first_seen | **date** | +| | last_seen | **date** | + +##### set: +| Key | Value | +| ------ | ------ | +| pgp_key:*key id* | *item_path* | +| | | +| pgp_name:*name* | *item_path* | +| | | +| pgp_mail:*mail* | *item_path* | + +##### Hset date: +| Key | Field | Value | +| ------ | ------ | +| pgp:key:*date* | *key* | *nb seen* | +| | | +| pgp:name:*date* | *name* | *nb seen* | +| | | +| pgp:mail:*date* | *mail* | *nb seen* | + +##### zset: +| Key | Field | Value | +| ------ | ------ | ------ | +| pgp_all:key | *key* | *nb seen* | +| | | +| pgp_all:name | *name* | *nb seen* | +| | | +| pgp_all:mail | *mail* | *nb seen* | + +##### set: +| Key | Value | +| ------ | ------ | +| item_pgp_key:*item_path* | *key* | +| | | +| item_pgp_name:*item_path* | *name* | +| | | +| item_pgp_mail:*item_path* | *mail* | + ## DB9 - Crawler: ##### Hset: diff --git a/bin/Keys.py b/bin/Keys.py index 7fcc7a1e..6d6e6166 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -108,6 +108,15 @@ def search_key(paste): p.populate_set_out(msg, 'Tags') find = True + if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in content: + p.populate_set_out(message, 'PgpDump') + + if '-----BEGIN PGP SIGNATURE-----' in content: + p.populate_set_out(message, 'PgpDump') + + if '-----BEGIN PGP MESSAGE-----' in content: + p.populate_set_out(message, 'PgpDump') + if find : #Send to duplicate diff --git a/bin/PgpDump.py b/bin/PgpDump.py new file mode 100755 index 00000000..6cf754d0 --- /dev/null +++ b/bin/PgpDump.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" + PgpDum module + + Extract ID from PGP Blocks +""" + +import os +import re +import redis +import signal +import datetime +import subprocess + +from pubsublogger import publisher +from bs4 import BeautifulSoup + +from Helper import Process +from packages import Paste + +class TimeoutException(Exception): + pass + +def timeout_handler(signum, frame): + raise TimeoutException + +signal.signal(signal.SIGALRM, timeout_handler) + +def remove_html(item_content): + if bool(BeautifulSoup(item_content, "html.parser").find()): + soup = BeautifulSoup(item_content, 'html.parser') + # kill all script and style elements + for script in soup(["script", "style"]): + script.extract() # remove + + # get text + text = soup.get_text() + return text + else: + return item_content + +def extract_all_id(item_content, regex): + # max execution time on regex + signal.alarm(max_execution_time) + try: + pgp_extracted_block = re.findall(regex, item_content) + except TimeoutException: + pgp_extracted_block = [] + p.incr_module_timeout_statistic() # add encoder type + print ("{0} processing timeout".format(paste.p_rel_path)) + else: + signal.alarm(0) + + for pgp_to_dump in pgp_extracted_block: + pgp_packet = get_pgp_packet(pgp_to_dump) + extract_id_from_output(pgp_packet) + +def get_pgp_packet(save_path): + save_path = '{}'.format(save_path) + process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE) + process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE) + process1.stdout.close() + output = process2.communicate()[0].decode() + return output + + +def extract_id_from_output(pgp_dump_outpout): + all_user_id = set(re.findall(regex_user_id, pgp_dump_outpout)) + for user_id in all_user_id: + user_id = user_id.replace(user_id_str, '', 1) + mail = None + if ' <' in user_id: + name, mail = user_id.rsplit(' <', 1) + mail = mail[:-1] + set_name.add(name) + set_mail.add(mail) + else: + name = user_id + set_name.add(name) + + all_key_id = set(re.findall(regex_key_id, pgp_dump_outpout)) + for key_id in all_key_id: + key_id = key_id.replace(key_id_str, '', 1) + set_key.add(key_id) + +def save_pgp_data(type_pgp, date, item_path, data): + # create basic medata + if not serv_metadata.exists('pgp_{}:{}'.format(type_pgp, data)): + serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date) + serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) + else: + last_seen = serv_metadata.hget('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen') + if not last_seen: + serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) + else: + if int(last_seen) < int(date): + serv_metadata.hset('pgp_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) + + # global set + serv_metadata.sadd('pgp_{}:{}'.format(type_pgp, data), item_path) + + # daily + serv_metadata.hincrby('pgp:{}:{}'.format(type_pgp, date), data, 1) + + # all type + serv_metadata.zincrby('pgp_all:{}'.format(type_pgp), data, 1) + + # item_metadata + serv_metadata.sadd('item_pgp_{}:{}'.format(type_pgp, item_path), data) + + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + #config_section = 'PgpDump' + config_section = 'PgpDump' + + # Setup the I/O queues + p = Process(config_section) + + serv_metadata = redis.StrictRedis( + host=p.config.get("ARDB_Metadata", "host"), + port=p.config.getint("ARDB_Metadata", "port"), + db=p.config.getint("ARDB_Metadata", "db"), + decode_responses=True) + + # Sent to the logging a description of the module + publisher.info("PgpDump started") + + user_id_str = 'User ID - ' + regex_user_id= '{}.+'.format(user_id_str) + + key_id_str = 'Key ID - ' + regex_key_id = '{}.+'.format(key_id_str) + regex_pgp_public_blocs = '-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' + regex_pgp_signature = '-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' + regex_pgp_message = '-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' + + re.compile(regex_user_id) + re.compile(regex_key_id) + re.compile(regex_pgp_public_blocs) + re.compile(regex_pgp_signature) + re.compile(regex_pgp_message) + + max_execution_time = p.config.getint("PgpDump", "max_execution_time") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + + set_key = set() + set_name = set() + set_mail = set() + paste = Paste.Paste(message) + + # Do something with the message from the queue + date = str(paste._get_p_date()) + content = paste.get_p_content() + content = remove_html(content) + + extract_all_id(content, regex_pgp_public_blocs) + extract_all_id(content, regex_pgp_signature) + extract_all_id(content, regex_pgp_message) + + for key_id in set_key: + print(key_id) + save_pgp_data('key', date, message, key_id) + + for name_id in set_name: + print(name_id) + save_pgp_data('name', date, message, name_id) + + for mail_id in set_mail: + print(mail_id) + save_pgp_data('mail', date, message, mail_id) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index f9483476..ea0ea55c 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -71,6 +71,9 @@ max_execution_time = 90 [Onion] max_execution_time = 180 +[PgpDump] +max_execution_time = 60 + [Base64] path = Base64/ max_execution_time = 60 diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index c3c09a4e..f1fe5e3d 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -118,6 +118,10 @@ publish = Redis_Duplicate,Redis_Tags [Keys] subscribe = Redis_Global +publish = Redis_Duplicate,Redis_PgpDump,Redis_Tags + +[PgpDump] +subscribe = Redis_PgpDump publish = Redis_Duplicate,Redis_Tags [ApiKey] From 6092f482e6037a76b28b067ebe4be425772e2ef3 Mon Sep 17 00:00:00 2001 From: kovacsbalu Date: Wed, 15 May 2019 09:57:18 +0200 Subject: [PATCH 10/33] Fix crawler rotation Before this, crawler processed prioritized onions and after all starts prioritized regular. --- bin/Crawler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/Crawler.py b/bin/Crawler.py index d8e6a430..31f2d594 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -12,6 +12,7 @@ import time import subprocess import requests +from collections import deque from pyfaup.faup import Faup sys.path.append(os.environ['AIL_BIN']) @@ -303,7 +304,7 @@ if __name__ == '__main__': #mode = sys.argv[1] splash_port = sys.argv[1] - rotation_mode = ['onion', 'regular'] + rotation_mode = deque(["onion", "regular"]) default_proto_map = {'http': 80, 'https': 443} ######################################################## add ftp ??? @@ -361,6 +362,7 @@ if __name__ == '__main__': update_auto_crawler() + rotation_mode.rotate() to_crawl = get_elem_to_crawl(rotation_mode) if to_crawl: url_data = unpack_url(to_crawl['url']) From 7765ab92e0d2a171cadc30b48f6c0560b1e908c2 Mon Sep 17 00:00:00 2001 From: kovacsbalu Date: Wed, 15 May 2019 10:00:51 +0200 Subject: [PATCH 11/33] Hopp, single quote :) --- bin/Crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/Crawler.py b/bin/Crawler.py index 31f2d594..e5864059 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -304,7 +304,7 @@ if __name__ == '__main__': #mode = sys.argv[1] splash_port = sys.argv[1] - rotation_mode = deque(["onion", "regular"]) + rotation_mode = deque(['onion', 'regular']) default_proto_map = {'http': 80, 'https': 443} ######################################################## add ftp ??? From cca69def7196749e8db0d0e5da6843601c6d8f4f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 15 May 2019 17:23:58 +0200 Subject: [PATCH 12/33] chg: [decoded items] bootstrap 4 migration --- .../modules/hashDecoded/Flask_hashDecoded.py | 10 +- .../hashDecoded/templates/hashDecoded.html | 493 +++++++++--------- .../hashDecoded/templates/showHash.html | 322 ++++++------ var/www/templates/decoded/menu_sidebar.html | 30 ++ var/www/templates/nav_bar.html | 2 +- 5 files changed, 449 insertions(+), 408 deletions(-) create mode 100644 var/www/templates/decoded/menu_sidebar.html diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 8a7945d2..1c5f8e89 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -66,13 +66,13 @@ def get_file_icon(estimated_type): if file_type == 'application': file_icon = 'fa-file ' elif file_type == 'audio': - file_icon = 'fa-file-video-o ' + file_icon = 'fa-file-audio ' elif file_type == 'image': - file_icon = 'fa-file-image-o' + file_icon = 'fa-file-image' elif file_type == 'text': - file_icon = 'fa-file-text-o' + file_icon = 'fa-file-alt' else: - file_icon = 'fa-file-o' + file_icon = 'fa-sticky-note' return file_icon @@ -88,7 +88,7 @@ def get_file_icon_text(estimated_type): elif file_type == 'text': file_icon_text = '\uf15c' else: - file_icon_text = '\uf15b' + file_icon_text = '\uf249' return file_icon_text diff --git a/var/www/modules/hashDecoded/templates/hashDecoded.html b/var/www/modules/hashDecoded/templates/hashDecoded.html index 19ddcdaf..bf0a12bf 100644 --- a/var/www/modules/hashDecoded/templates/hashDecoded.html +++ b/var/www/modules/hashDecoded/templates/hashDecoded.html @@ -1,157 +1,147 @@ - - - + + Decoded - AIL + - HashesDecoded - AIL + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + - {% include 'navbar.html' %} + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'decoded/menu_sidebar.html' %} + +
-
-
-

Hashed Files

-
- -
-
- - -
-
-
-
-
- -
-
-
- Select a date range : -
-
- - -
-
- - -
- Encoding : - - File Type : - -
- -
- - -
-
-
-
-
-
-
+
+
- - {% if l_64|length != 0 %} - {% if date_from|string == date_to|string %} -

{{ date_from }} Hashed files:

- {% else %} -

{{ date_from }} to {{ date_to }} Hashed files:

- {% endif %} - - +
+ +
+
+
Select a date range :
+ +
+
+ +
+
+
+ +
+
Encoding :
+ +
File Type :
+ +
+ + +
+ + +
+
+ +
+
+
+
+
+ + + {% if l_64|length != 0 %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Decoded files:

+ {% else %} +

{{ date_from }} to {{ date_to }} Decoded files:

+ {% endif %} +
+ @@ -162,121 +152,136 @@ - - - {% for b64 in l_64 %} - - - - - - - - + {% for b64 in l_64 %} + + + + + + + + - - - {% endfor %} - -
estimated type hashVirus Total Sparkline
  {{ b64[1] }}{{ b64[2] }}{{ b64[5] }}{{ b64[6] }}{{ b64[3] }}{{ b64[4] }} - {% if vt_enabled %} - {% if not b64[7] %} - - - - {% else %} - VT Report - {% endif %} -
  {{ b64[1] }}{{ b64[2] }}{{ b64[5] }}{{ b64[6] }}{{ b64[3] }}{{ b64[4] }} + {% if vt_enabled %} + {% if not b64[7] %} + + - {% else %} - Virus Total submission is disabled - {% endif %} + + {% else %} +  VT Report + {% endif %} + + {% else %} + Virus Total submission is disabled + {% endif %} - -
- {% else %} - {% if show_decoded_files %} - {% if date_from|string == date_to|string %} -

{{ date_from }}, No Hashes

- {% else %} -

{{ date_from }} to {{ date_to }}, No Hashes

- {% endif %} + + + + {% endfor %} + + + {% else %} + {% if show_decoded_files %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Hashes

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Hashes

{% endif %} {% endif %} -
- + {% endif %}
- - - chart.onResize(); - $(window).on("resize", function() { - chart.onResize(); - }); - }); - - + diff --git a/var/www/modules/hashDecoded/templates/showHash.html b/var/www/modules/hashDecoded/templates/showHash.html index b0bcaa50..45faf5c8 100644 --- a/var/www/modules/hashDecoded/templates/showHash.html +++ b/var/www/modules/hashDecoded/templates/showHash.html @@ -3,27 +3,20 @@ - Hash Information - AIL + - - - - - + + + - - - - - - - - - + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'decoded/menu_sidebar.html' %} + +
+ +
+
+
+
+
+ +
+ +
+
+
Select a date range :
+ +
+
+ +
+
+
+ +
+
Type ID :
+ +
+ + +
+ + +
+
+ +
+
+
+
+
+
+ + {% if l_pgpdump|length != 0 %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Dumped Keys:

+ {% else %} +

{{ date_from }} to {{ date_to }} Dumped Keys:

+ {% endif %} + + + + + + + + + + + + + {% for pgp_dump in l_pgpdump %} + + + + + + + + + {% endfor %} + +
type idkey idfirst seenlast seennb pasteSparkline
  {{ l_pgpdump[pgp_dump]['type_id'] }}{{ pgp_dump }}{{ l_pgpdump[pgp_dump]['first_seen'] }}{{ l_pgpdump[pgp_dump]['last_seen'] }}{{ l_pgpdump[pgp_dump]['nb_seen'] }}
+ {% else %} + {% if show_decoded_files %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Dumped Keys

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Dumped Keys

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+
+ + + + + + + + + + + + + + + + + diff --git a/var/www/modules/hashDecoded/templates/hashDecoded.html b/var/www/modules/hashDecoded/templates/hashDecoded.html index bf0a12bf..e031530a 100644 --- a/var/www/modules/hashDecoded/templates/hashDecoded.html +++ b/var/www/modules/hashDecoded/templates/hashDecoded.html @@ -493,16 +493,19 @@ function removePopovers () { function showPopover (d) { $(this).popover({ - title: d.name, + title: "", placement: 'top', container: 'body', trigger: 'manual', html : true, content: function() { - return d.label + - "
num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); } + return "" + + "
num: "; } }); - $(this).popover('show') + $(this).popover('show'); + $("#tooltip-id-name-bar").text(d.name); + $("#tooltip-id-label").text(d.label); + $("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0)); } chart.onResize = function () { @@ -571,13 +574,16 @@ function draw_pie_chart(id, url_json, pie_on_click_url) { function mouseovered_pie(d) { + //remove old content + $("#tooltip-id-name").remove(); + $("#tooltip-id-value").remove(); // tooltip var content; - content = ""+d.data.name+""+"
"+ + content = "
"+ "
"+ - "Decoded: "+d.data.value+"
" + "Decoded:
" div_pie.transition() .duration(200) @@ -585,6 +591,9 @@ function draw_pie_chart(id, url_json, pie_on_click_url) { div_pie.html(content) .style("left", (d3.event.pageX) + "px") .style("top", (d3.event.pageY - 28) + "px"); + + $("#tooltip-id-name").text(d.data.name); + $("#tooltip-id-value").text(d.data.value); } function mouseouted_pie() { diff --git a/var/www/modules/hashDecoded/templates/showHash.html b/var/www/modules/hashDecoded/templates/showHash.html index 45faf5c8..c7cac4d8 100644 --- a/var/www/modules/hashDecoded/templates/showHash.html +++ b/var/www/modules/hashDecoded/templates/showHash.html @@ -171,8 +171,8 @@
{% for encoding in list_hash_decoder %} - {% endfor %}
@@ -398,8 +398,7 @@ d3.json(url) node.append('text') .attr('text-anchor', 'middle') .attr('dominant-baseline', 'central') - .attr("class", "graph_node_icon") - .attr('font-family', 'FontAwesome') + .attr("class", "graph_node_icon fa") .attr('font-size', '8px' ) .attr('pointer-events', 'none') .text(function(d) { @@ -475,24 +474,36 @@ function mouseovered(d) { var content; if(d.hash == true){ - content = ""+d.id+""+"
"+ + content = "
"+ "
"+ - "First seen: "+d.first_seen+"
"+ - "Last seen: "+d.last_seen+"
"+ - "nb_seen_in_paste: "+d.nb_seen_in_paste+"
"+ - "Size (kb): "+d.size+"
"+ - "
"+ - "Estimated type: "+d.estimated_type; -} else { - content = ""+d.id+""+"
"; -} + "First seen:
"+ + "Last seen:
"+ + "nb_seen: "; div.transition() - .duration(200) - .style("opacity", .9); + .duration(200) + .style("opacity", .9); div.html(content) - .style("left", (d3.event.pageX) + "px") - .style("top", (d3.event.pageY - 28) + "px"); + .style("left", (d3.event.pageX) + "px") + .style("top", (d3.event.pageY - 28) + "px"); + + $("#tooltip-id-name").text(d.id); + $("#tooltip-id-first_seen").text(d.first_seen); + $("#tooltip-id-last_seen").text(d.last_seen); + $("#tooltip-id-nb_seen").text(d.nb_seen_in_paste); + +} else { + content = "
"; + + div.transition() + .duration(200) + .style("opacity", .9); + div.html(content) + .style("left", (d3.event.pageX) + "px") + .style("top", (d3.event.pageY - 28) + "px"); + + $("#tooltip-id-name").text(d.id); +} //links /*link.style("stroke-opacity", function(o) { diff --git a/var/www/modules/hashDecoded/templates/showPgpDump.html b/var/www/modules/hashDecoded/templates/showPgpDump.html new file mode 100644 index 00000000..8f7b05aa --- /dev/null +++ b/var/www/modules/hashDecoded/templates/showPgpDump.html @@ -0,0 +1,569 @@ + + + + + + + AIL - framework + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'decoded/menu_sidebar.html' %} + +
+ +
+
+

{{ key_id }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + +
    typeFirst_seenLast_seenNb seen
      {{ type_id }}{{ key_id_metadata['first_seen'] }}{{ key_id_metadata['last_seen'] }}{{ key_id_metadata['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • +
+
+
+ +
+
+ +
+
+ Graph +
+
+
+
+
+
+
+ +
+ +
+
+ Graph +
+
+ + + +
    +
    • +
  • +

    Double click on a node to open Hash/Paste

    + + Current Hash
    + + Hashes
    + + Pastes +

    +
    • +
  • + Hash Types: +
    • +
  • + Key
    + Name
    + Mail
    +
    • +
+
+
+
+
+ +
+
+ Graph +
+
+
+
+
+
+ +
+
+
+ + + + + + + + + + + + + diff --git a/var/www/templates/decoded/menu_sidebar.html b/var/www/templates/decoded/menu_sidebar.html index 3cb5868b..48fefd56 100644 --- a/var/www/templates/decoded/menu_sidebar.html +++ b/var/www/templates/decoded/menu_sidebar.html @@ -8,9 +8,6 @@
+ + + + + + + + + + + + + + + + + diff --git a/var/www/modules/hashDecoded/templates/showCorrelation.html b/var/www/modules/hashDecoded/templates/showCorrelation.html new file mode 100644 index 00000000..35ec63ec --- /dev/null +++ b/var/www/modules/hashDecoded/templates/showCorrelation.html @@ -0,0 +1,553 @@ + + + + + + + AIL - framework + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'decoded/menu_sidebar.html' %} + +
+ +
+
+

{{ key_id }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + +
    typeFirst_seenLast_seenNb seen
      {{ type_id }}{{ key_id_metadata['first_seen'] }}{{ key_id_metadata['last_seen'] }}{{ key_id_metadata['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • +
+
+
+ +
+
+ +
+
+ Graph +
+
+
+
+
+
+
+ +
+ +
+
+ Graph +
+
+ + + {% if correlation_type=='pgpdump' %} + {% include 'decoded/show_helper_pgpdump.html' %} + {% elif correlation_type=='cryptocurrency' %} + {% include 'decoded/show_helper_cryptocurrency.html' %} + {% endif %} + +
+
+
+
+ +
+
+ Graph +
+
+
+
+
+
+ +
+
+
+ + + + + + + + + + + + + diff --git a/var/www/templates/decoded/show_helper_cryptocurrency.html b/var/www/templates/decoded/show_helper_cryptocurrency.html new file mode 100644 index 00000000..7ca07720 --- /dev/null +++ b/var/www/templates/decoded/show_helper_cryptocurrency.html @@ -0,0 +1,20 @@ +
    +
    • +
  • +

    Double click on a node to open Cryptocurrency/Paste

    + + Current Cryptocurrency
    + + Cryptocurrency
    + + Pastes +

    +
    • +
  • + Cryptocurrency Types: +
    • +
  • + Bitcoin
    + Monero
    +
    • +
diff --git a/var/www/templates/decoded/show_helper_pgpdump.html b/var/www/templates/decoded/show_helper_pgpdump.html new file mode 100644 index 00000000..1dbf42ba --- /dev/null +++ b/var/www/templates/decoded/show_helper_pgpdump.html @@ -0,0 +1,21 @@ +
    +
    • +
  • +

    Double click on a node to open PgpDump/Paste

    + + Current PgpDump
    + + PgpDump
    + + Pastes +

    +
    • +
  • + PgpDump Types: +
    • +
  • + Key
    + Name
    + Mail
    +
    • +
From ffc10decc6bee938719faa187e1a4bdb2a17e0f4 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 May 2019 13:04:00 +0200 Subject: [PATCH 21/33] chg: [update v1.7] update thirdparty --- update/v1.7/Update.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/update/v1.7/Update.sh b/update/v1.7/Update.sh index 8eb6c5a1..601b3cab 100755 --- a/update/v1.7/Update.sh +++ b/update/v1.7/Update.sh @@ -49,6 +49,12 @@ echo -e $GREEN"Shutting down ARDB ..."$DEFAULT bash ${AIL_BIN}/LAUNCH.sh -k & wait +echo "" +echo -e $GREEN"Update thirdparty ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -t & +wait + + echo "" exit 0 From eb9c81acef5bce3f749b49ef8fc5ba5f019f9f26 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 May 2019 13:08:10 +0200 Subject: [PATCH 22/33] chg: [correlation] clean files --- bin/helper/reprocess_bitcoin.py | 50 ++ .../hashDecoded/templates/PgpDecoded.html | 660 ------------------ .../hashDecoded/templates/showPgpDump.html | 569 --------------- var/www/templates/decoded/menu_sidebar.html | 15 +- 4 files changed, 63 insertions(+), 1231 deletions(-) create mode 100755 bin/helper/reprocess_bitcoin.py delete mode 100644 var/www/modules/hashDecoded/templates/PgpDecoded.html delete mode 100644 var/www/modules/hashDecoded/templates/showPgpDump.html diff --git a/bin/helper/reprocess_bitcoin.py b/bin/helper/reprocess_bitcoin.py new file mode 100755 index 00000000..5e609f1f --- /dev/null +++ b/bin/helper/reprocess_bitcoin.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import gzip +import base64 +import uuid +import datetime +import base64 +import redis +import json +import time + +sys.path.append(os.environ['AIL_BIN']) +from Helper import Process + +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + +config_section = 'Global' +p = Process(config_section) + +r_tags = redis.StrictRedis( + host=p.config.get("ARDB_Tags", "host"), + port=p.config.getint("ARDB_Tags", "port"), + db=p.config.getint("ARDB_Tags", "db"), + decode_responses=True) + +tag = 'infoleak:automatic-detection="bitcoin-address"' + +# get tag first/last seen +first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen') +last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') + +l_dates = substract_date(first_seen, last_seen) + +# get all tagged items +for date in l_dates: + daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date)) + + for item in daily_tagged_items: + p.populate_set_out(item) diff --git a/var/www/modules/hashDecoded/templates/PgpDecoded.html b/var/www/modules/hashDecoded/templates/PgpDecoded.html deleted file mode 100644 index 88da78e0..00000000 --- a/var/www/modules/hashDecoded/templates/PgpDecoded.html +++ /dev/null @@ -1,660 +0,0 @@ - - - - - Decoded - AIL - - - - - - - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'decoded/menu_sidebar.html' %} - -
- -
-
-
-
-
- -
- -
-
-
Select a date range :
- -
-
- -
-
-
- -
-
Type ID :
- -
- - -
- - -
-
- -
-
-
-
-
-
- - {% if l_pgpdump|length != 0 %} - {% if date_from|string == date_to|string %} -

{{ date_from }} Dumped Keys:

- {% else %} -

{{ date_from }} to {{ date_to }} Dumped Keys:

- {% endif %} - - - - - - - - - - - - - {% for pgp_dump in l_pgpdump %} - - - - - - - - - {% endfor %} - -
type idkey idfirst seenlast seennb pasteSparkline
  {{ l_pgpdump[pgp_dump]['type_id'] }}{{ pgp_dump }}{{ l_pgpdump[pgp_dump]['first_seen'] }}{{ l_pgpdump[pgp_dump]['last_seen'] }}{{ l_pgpdump[pgp_dump]['nb_seen'] }}
- {% else %} - {% if show_decoded_files %} - {% if date_from|string == date_to|string %} -

{{ date_from }}, No Dumped Keys

- {% else %} -

{{ date_from }} to {{ date_to }}, No Dumped Keys

- {% endif %} - {% endif %} - {% endif %} -
- -
-
-
- - - - - - - - - - - - - - - - - diff --git a/var/www/modules/hashDecoded/templates/showPgpDump.html b/var/www/modules/hashDecoded/templates/showPgpDump.html deleted file mode 100644 index 8f7b05aa..00000000 --- a/var/www/modules/hashDecoded/templates/showPgpDump.html +++ /dev/null @@ -1,569 +0,0 @@ - - - - - - - AIL - framework - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'decoded/menu_sidebar.html' %} - -
- -
-
-

{{ key_id }} :

-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - -
    typeFirst_seenLast_seenNb seen
      {{ type_id }}{{ key_id_metadata['first_seen'] }}{{ key_id_metadata['last_seen'] }}{{ key_id_metadata['nb_seen'] }}
    -
    -
    -
    -
    -
    -
    • -
-
-
- -
-
- -
-
- Graph -
-
-
-
-
-
-
- -
- -
-
- Graph -
-
- - - -
    -
    • -
  • -

    Double click on a node to open Hash/Paste

    - - Current Hash
    - - Hashes
    - - Pastes -

    -
    • -
  • - Hash Types: -
    • -
  • - Key
    - Name
    - Mail
    -
    • -
-
-
-
-
- -
-
- Graph -
-
-
-
-
-
- -
-
-
- - - - - - - - - - - - - diff --git a/var/www/templates/decoded/menu_sidebar.html b/var/www/templates/decoded/menu_sidebar.html index 7a5a981c..61c17a08 100644 --- a/var/www/templates/decoded/menu_sidebar.html +++ b/var/www/templates/decoded/menu_sidebar.html @@ -22,11 +22,22 @@ +
+ Cryptocurrency +
+ From bc6332dae4b77930eb8498934e0c3bd3c70479e0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 May 2019 13:19:11 +0200 Subject: [PATCH 23/33] fix: [correlation] fix endpoint --- var/www/templates/decoded/menu_sidebar.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/templates/decoded/menu_sidebar.html b/var/www/templates/decoded/menu_sidebar.html index 61c17a08..f8b63426 100644 --- a/var/www/templates/decoded/menu_sidebar.html +++ b/var/www/templates/decoded/menu_sidebar.html @@ -33,7 +33,7 @@
  • - + Cryptocurrency From c27dfc035a0090eb594fcb044800cdf8e6068957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Fri, 24 May 2019 13:32:50 +0200 Subject: [PATCH 24/33] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b6eab35..9f9d8d4d 100644 --- a/README.md +++ b/README.md @@ -62,11 +62,12 @@ Features * Create events on [MISP](https://github.com/MISP/MISP) and cases on [The Hive](https://github.com/TheHive-Project/TheHive) * Automatic paste export at detection on [MISP](https://github.com/MISP/MISP) (events) and [The Hive](https://github.com/TheHive-Project/TheHive) (alerts) on selected tags * Extracted and decoded files can be searched by date range, type of file (mime-type) and encoding discovered -* Graph relationships between decoded file (hashes) +* Graph relationships between decoded file (hashes), similar PGP UIDs and addresses of cryptocurrencies * Tor hidden services crawler to crawl and parse output * Tor onion availability is monitored to detect up and down of hidden services * Browser hidden services are screenshot and integrated in the analysed output including a blurring screenshot interface (to avoid "burning the eyes" of the security analysis with specific content) * Tor hidden services is part of the standard framework, all the AIL modules are available to the crawled hidden services +* Generic web crawler to trigger crawling on demand or at regular interval URL or Tor hidden services Installation From fae3530fec149ad2e97f472e7b94924702d69916 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 24 May 2019 14:10:06 +0200 Subject: [PATCH 25/33] chg: [PgpDump] PMF fix - longer keys are truncated TODO: Need to be updated by an async process handling the keys --- bin/PgpDump.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/PgpDump.py b/bin/PgpDump.py index b982b9c5..0dc43621 100755 --- a/bin/PgpDump.py +++ b/bin/PgpDump.py @@ -59,6 +59,9 @@ def extract_all_id(item_content, regex): def get_pgp_packet(save_path): save_path = '{}'.format(save_path) + print (len(save_path)) + if len(save_path) > 131072: + save_path = save_path[:131071] process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE) process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE) process1.stdout.close() From 8dfe8fec911eeb3217c9a3c9371d311b7821c229 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 May 2019 14:14:07 +0200 Subject: [PATCH 26/33] fix: [correlation] fix graph node endpoint --- var/www/modules/hashDecoded/Flask_hashDecoded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 597190c1..1cbf4f98 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -460,7 +460,7 @@ def correlation_graph_node_json(correlation_type, type_id, key_id): nodes = [] for node in nodes_set_dump: - nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_icon_text(correlation_type, node[2]),"url": url_for(get_show_key_id_endpoint(correlation_type), hash=node[0]), 'hash': True}) + nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_icon_text(correlation_type, node[2]),"url": url_for(get_show_key_id_endpoint(correlation_type), type_id=node[2], key_id=node[0]), 'hash': True}) for node in nodes_set_paste: nodes.append({"id": node[0], "group": node[1],"url": url_for('showsavedpastes.showsavedpaste', paste=node[0]), 'hash': False}) links = [] From 67897eeef150cce86b46870471b29edf9d833755 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 May 2019 14:31:08 +0200 Subject: [PATCH 27/33] fix: [update v1.7] add bs4 requirement --- requirements.txt | 2 ++ update/v1.7/Update.sh | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/requirements.txt b/requirements.txt index 3991e158..fe98901d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,6 +51,8 @@ DomainClassifier #Indexer requirements whoosh +beautifulsoup4 + ipaddress pycountry diff --git a/update/v1.7/Update.sh b/update/v1.7/Update.sh index 601b3cab..fe941541 100755 --- a/update/v1.7/Update.sh +++ b/update/v1.7/Update.sh @@ -32,6 +32,11 @@ sudo make install wait echo "" +echo "" +echo -e $GREEN"Update requirement"$DEFAULT +echo "" +pip3 install beautifulsoup4 + bash ${AIL_BIN}LAUNCH.sh -lav & wait echo "" From f09bb63a358197a0a843836e536d32422b1a018c Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sat, 25 May 2019 09:51:49 +0200 Subject: [PATCH 28/33] chg: [template] the "item" project --- var/www/modules/hashDecoded/templates/DaysCorrelation.html | 2 +- var/www/modules/hashDecoded/templates/hashDecoded.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/modules/hashDecoded/templates/DaysCorrelation.html b/var/www/modules/hashDecoded/templates/DaysCorrelation.html index 84868f22..2051cef0 100644 --- a/var/www/modules/hashDecoded/templates/DaysCorrelation.html +++ b/var/www/modules/hashDecoded/templates/DaysCorrelation.html @@ -136,7 +136,7 @@ key id first seen last seen - nb paste + nb item Sparkline diff --git a/var/www/modules/hashDecoded/templates/hashDecoded.html b/var/www/modules/hashDecoded/templates/hashDecoded.html index e031530a..766f618a 100644 --- a/var/www/modules/hashDecoded/templates/hashDecoded.html +++ b/var/www/modules/hashDecoded/templates/hashDecoded.html @@ -147,7 +147,7 @@ hash first seen last seen - nb paste + nb item size Virus Total Sparkline From 9cb6312279020024a6a3d00c0c0829bc30aaa711 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 4 Jun 2019 14:22:46 +0200 Subject: [PATCH 29/33] fix: [UI crawler endpoints] display crawler status + fix #353 --- var/www/Flask_server.py | 4 ---- var/www/modules/Flask_config.py | 2 ++ var/www/modules/hiddenServices/Flask_hiddenServices.py | 6 +++++- .../templates/Crawler_Splash_last_by_type.html | 2 ++ .../hiddenServices/templates/Crawler_Splash_manual.html | 1 + var/www/modules/hiddenServices/templates/Crawler_auto.html | 2 ++ .../modules/hiddenServices/templates/Crawler_dashboard.html | 2 ++ var/www/templates/crawler/crawler_disabled.html | 6 ++++++ 8 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 var/www/templates/crawler/crawler_disabled.html diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 95433757..ff6d5bac 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -48,10 +48,6 @@ except IOError: f = open('templates/ignored_modules.txt', 'w') f.close() -activate_crawler = cfg.get("Crawler", "activate_crawler") -if activate_crawler != 'True': - toIgnoreModule.add('hiddenServices') - # Dynamically import routes and functions from modules # Also, prepare header.html to_add_to_header_dico = {} diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index d4251e5f..899a26b5 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -173,6 +173,8 @@ REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git' max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs")) +crawler_enabled = cfg.getboolean("Crawler", "activate_crawler") + # VT try: from virusTotalKEYS import vt_key diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 0ee44974..ed5cbcd4 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -25,6 +25,7 @@ baseUrl = Flask_config.baseUrl r_cache = Flask_config.r_cache r_serv_onion = Flask_config.r_serv_onion r_serv_metadata = Flask_config.r_serv_metadata +crawler_enabled = Flask_config.crawler_enabled bootstrap_label = Flask_config.bootstrap_label hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates') @@ -247,6 +248,7 @@ def dashboard(): statDomains_regular = get_stats_last_crawled_domains('regular', date) return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion, + crawler_enabled=crawler_enabled, crawler_metadata_regular=crawler_metadata_regular, statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular) @@ -256,7 +258,7 @@ def hiddenServices_page_test(): @hiddenServices.route("/crawlers/manual", methods=['GET']) def manual(): - return render_template("Crawler_Splash_manual.html") + return render_template("Crawler_Splash_manual.html", crawler_enabled=crawler_enabled) @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET']) def crawler_splash_onion(): @@ -294,6 +296,7 @@ def Crawler_Splash_last_by_type(): crawler_metadata = get_crawler_splash_status(type) return render_template("Crawler_Splash_last_by_type.html", type=type, type_name=type_name, + crawler_enabled=crawler_enabled, last_domains=list_domains, statDomains=statDomains, crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string) @@ -507,6 +510,7 @@ def auto_crawler(): return render_template("Crawler_auto.html", page=page, nb_page_max=nb_page_max, last_domains=last_domains, + crawler_enabled=crawler_enabled, auto_crawler_domain_onions_metadata=auto_crawler_domain_onions_metadata, auto_crawler_domain_regular_metadata=auto_crawler_domain_regular_metadata) diff --git a/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html b/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html index a42e3880..bafe3ecf 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html +++ b/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html @@ -46,6 +46,8 @@
    + {% include 'crawler/crawler_disabled.html' %} +
    diff --git a/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html b/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html index bc2e6024..11df7730 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html +++ b/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html @@ -30,6 +30,7 @@
    + {% include 'crawler/crawler_disabled.html' %}
    diff --git a/var/www/modules/hiddenServices/templates/Crawler_auto.html b/var/www/modules/hiddenServices/templates/Crawler_auto.html index 977aed1e..87dd7569 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_auto.html +++ b/var/www/modules/hiddenServices/templates/Crawler_auto.html @@ -29,6 +29,8 @@
    + {% include 'crawler/crawler_disabled.html' %} + {%if last_domains%}
    diff --git a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html index 91d300af..b746b05c 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html +++ b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html @@ -26,6 +26,8 @@
    + {% include 'crawler/crawler_disabled.html' %} +
    diff --git a/var/www/templates/crawler/crawler_disabled.html b/var/www/templates/crawler/crawler_disabled.html new file mode 100644 index 00000000..455f350e --- /dev/null +++ b/var/www/templates/crawler/crawler_disabled.html @@ -0,0 +1,6 @@ +{% if not crawler_enabled %} + +{% endif %} From 1929b2c6a797944cf3f94914b80dc675624aa016 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 5 Jun 2019 16:18:30 +0200 Subject: [PATCH 30/33] fix: [paste_submit UI] filter empty file field --- var/www/modules/PasteSubmit/Flask_PasteSubmit.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index cc38de77..68cd75ae 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -294,6 +294,12 @@ def submit(): ltagsgalaxies = request.form['tags_galaxies'] paste_content = request.form['paste_content'] + is_file = False + file = request.files['file'] + if file: + if file.filename: + is_file = True + submitted_tag = 'infoleak:submission="manual"' #active taxonomies @@ -312,9 +318,7 @@ def submit(): else: ltags = submitted_tag - if 'file' in request.files: - - file = request.files['file'] + if is_file: if file: if file and allowed_file(file.filename): From 17af705231592951ac3190cd4999d7a7c7a9d427 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 5 Jun 2019 16:20:26 +0200 Subject: [PATCH 31/33] fix: [PgpDump] process large pgp blocks --- bin/PgpDump.py | 125 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 86 insertions(+), 39 deletions(-) diff --git a/bin/PgpDump.py b/bin/PgpDump.py index 0dc43621..21ffd263 100755 --- a/bin/PgpDump.py +++ b/bin/PgpDump.py @@ -9,6 +9,7 @@ import os import re import time +import uuid import redis import signal import datetime @@ -28,6 +29,15 @@ def timeout_handler(signum, frame): signal.signal(signal.SIGALRM, timeout_handler) +# save pgp message in directory, process one time each day +def save_in_file(message, pgp_content): + print('save in file') + UUID = str(uuid.uuid4()) + file_name = os.path.join(pgp_dump_dir_to_process, UUID) + with open(file_name, 'w') as f: + f.write(pgp_content) + r_serv_db.sadd('pgpdumb:uuid', '{};{}'.format(UUID, message)) + def remove_html(item_content): if bool(BeautifulSoup(item_content, "html.parser").find()): soup = BeautifulSoup(item_content, 'html.parser') @@ -41,33 +51,44 @@ def remove_html(item_content): else: return item_content -def extract_all_id(item_content, regex): - # max execution time on regex - signal.alarm(max_execution_time) - try: - pgp_extracted_block = re.findall(regex, item_content) - except TimeoutException: - pgp_extracted_block = [] - p.incr_module_timeout_statistic() # add encoder type - print ("{0} processing timeout".format(paste.p_rel_path)) - else: - signal.alarm(0) +def extract_all_id(message, item_content, regex=None, is_file=False): - for pgp_to_dump in pgp_extracted_block: - pgp_packet = get_pgp_packet(pgp_to_dump) + if is_file: + pgp_packet = get_pgp_packet_file(item_content) extract_id_from_output(pgp_packet) -def get_pgp_packet(save_path): - save_path = '{}'.format(save_path) - print (len(save_path)) - if len(save_path) > 131072: - save_path = save_path[:131071] - process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE) - process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE) - process1.stdout.close() - output = process2.communicate()[0].decode() - return output + else: + # max execution time on regex + signal.alarm(max_execution_time) + try: + pgp_extracted_block = re.findall(regex, item_content) + except TimeoutException: + pgp_extracted_block = [] + p.incr_module_timeout_statistic() # add encoder type + print ("{0} processing timeout".format(paste.p_rel_path)) + else: + signal.alarm(0) + for pgp_to_dump in pgp_extracted_block: + pgp_packet = get_pgp_packet(message, pgp_to_dump) + extract_id_from_output(pgp_packet) + +def get_pgp_packet(message, save_path): + save_path = '{}'.format(save_path) + if len(save_path) > 131072: + save_in_file(message, save_path) + return '' + else: + process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE) + process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE) + process1.stdout.close() + output = process2.communicate()[0].decode() + return output + +def get_pgp_packet_file(file): + process1 = subprocess.Popen([ 'pgpdump', file], stdout=subprocess.PIPE) + output = process1.communicate()[0].decode() + return output def extract_id_from_output(pgp_dump_outpout): all_user_id = set(re.findall(regex_user_id, pgp_dump_outpout)) @@ -131,6 +152,12 @@ if __name__ == '__main__': # Setup the I/O queues p = Process(config_section) + r_serv_db = redis.StrictRedis( + host=p.config.get("ARDB_DB", "host"), + port=p.config.getint("ARDB_DB", "port"), + db=p.config.getint("ARDB_DB", "db"), + decode_responses=True) + serv_metadata = redis.StrictRedis( host=p.config.get("ARDB_Metadata", "host"), port=p.config.getint("ARDB_Metadata", "port"), @@ -140,6 +167,11 @@ if __name__ == '__main__': # Sent to the logging a description of the module publisher.info("PgpDump started") + # check/create pgpdump queue directory (used for huge pgp blocks) + pgp_dump_dir_to_process = os.path.join(os.environ['AIL_HOME'], 'temp', 'pgpdump') + if not os.path.isdir(pgp_dump_dir_to_process): + os.makedirs(pgp_dump_dir_to_process) + user_id_str = 'User ID - ' regex_user_id= '{}.+'.format(user_id_str) @@ -159,28 +191,43 @@ if __name__ == '__main__': # Endless loop getting messages from the input queue while True: - # Get one message from the input queue - message = p.get_from_set() - - if message is None: - publisher.debug("{} queue is empty, waiting".format(config_section)) - time.sleep(1) - continue - + is_file = False set_key = set() set_name = set() set_mail = set() - paste = Paste.Paste(message) - # Do something with the message from the queue - date = str(paste._get_p_date()) - content = paste.get_p_content() - content = remove_html(content) + if r_serv_db.scard('pgpdumb:uuid') > 0: + res = r_serv_db.spop('pgpdumb:uuid') + file_to_process, message = res.split(';', 1) + file_to_process = os.path.join(pgp_dump_dir_to_process, file_to_process) + date = datetime.datetime.now().strftime("%Y/%m/%d") + paste = Paste.Paste(message) + date = str(paste._get_p_date()) + print(message) + extract_all_id(message, file_to_process, is_file=True) + os.remove(file_to_process) - extract_all_id(content, regex_pgp_public_blocs) - extract_all_id(content, regex_pgp_signature) - extract_all_id(content, regex_pgp_message) + else: + # Get one message from the input queue + message = p.get_from_set() + + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + paste = Paste.Paste(message) + + date = str(paste._get_p_date()) + content = paste.get_p_content() + content = remove_html(content) + + print(message) + + extract_all_id(message, content, regex_pgp_public_blocs) + extract_all_id(message, content, regex_pgp_signature) + extract_all_id(message, content, regex_pgp_message) for key_id in set_key: print(key_id) From 601ea8f13e4d972a17d86af7cf75b6f469a1f7af Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 5 Jun 2019 16:31:35 +0200 Subject: [PATCH 32/33] chg: [paste_submit UI] add debug --- var/www/modules/PasteSubmit/Flask_PasteSubmit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 68cd75ae..51b5f2b2 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -310,6 +310,7 @@ def submit(): if ltags or ltagsgalaxies: if not addTagsVerification(ltags, ltagsgalaxies): content = 'INVALID TAGS' + print(content) return content, 400 # add submitted tags @@ -358,6 +359,7 @@ def submit(): else: content = 'wrong file type, allowed_extensions: sh, pdf, zip, gz, tar.gz or remove the extension' + print(content) return content, 400 @@ -380,9 +382,11 @@ def submit(): else: content = 'size error' + print(content) return content, 400 content = 'submit aborded' + print(content) return content, 400 From 0d3d4aae1d14ed0973bb99e689f28c20668eeff0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 5 Jun 2019 16:41:59 +0200 Subject: [PATCH 33/33] fix: [paste_submit UI] filter empty file field --- var/www/modules/PasteSubmit/Flask_PasteSubmit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 51b5f2b2..6b7f5360 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -295,10 +295,11 @@ def submit(): paste_content = request.form['paste_content'] is_file = False - file = request.files['file'] - if file: - if file.filename: - is_file = True + if 'file' in request.files: + file = request.files['file'] + if file: + if file.filename: + is_file = True submitted_tag = 'infoleak:submission="manual"'