mirror of https://github.com/CIRCL/AIL-framework
Merge branch 'master' into user_management
Terrtia
commit
99e35c51ec
82
OVERVIEW.md
82
OVERVIEW.md
|
|
@ -168,6 +168,88 @@ Redis and ARDB overview
|
|||
| binary_hash:**hash** | **item** | **nb_seen** |
|
||||
| hexadecimal_hash:**hash** | **item** | **nb_seen** |
|
||||
|
||||
#### PgpDump
|
||||
|
||||
##### Hset:
|
||||
| Key | Field | Value |
|
||||
| ------ | ------ | ------ |
|
||||
| pgpdump_metadata_key:*key id* | first_seen | **date** |
|
||||
| | last_seen | **date** |
|
||||
| | |
|
||||
| pgpdump_metadata_name:*name* | first_seen | **date** |
|
||||
| | last_seen | **date** |
|
||||
| | |
|
||||
| pgpdump_metadata_mail:*mail* | first_seen | **date** |
|
||||
| | last_seen | **date** |
|
||||
|
||||
##### set:
|
||||
| Key | Value |
|
||||
| ------ | ------ |
|
||||
| set_pgpdump_key:*key id* | *item_path* |
|
||||
| | |
|
||||
| set_pgpdump_name:*name* | *item_path* |
|
||||
| | |
|
||||
| set_pgpdump_mail:*mail* | *item_path* |
|
||||
|
||||
##### Hset date:
|
||||
| Key | Field | Value |
|
||||
| ------ | ------ |
|
||||
| pgpdump:key:*date* | *key* | *nb seen* |
|
||||
| | |
|
||||
| pgpdump:name:*date* | *name* | *nb seen* |
|
||||
| | |
|
||||
| pgpdump:mail:*date* | *mail* | *nb seen* |
|
||||
|
||||
##### zset:
|
||||
| Key | Field | Value |
|
||||
| ------ | ------ | ------ |
|
||||
| pgpdump_all:key | *key* | *nb seen* |
|
||||
| | |
|
||||
| pgpdump_all:name | *name* | *nb seen* |
|
||||
| | |
|
||||
| pgpdump_all:mail | *mail* | *nb seen* |
|
||||
|
||||
##### set:
|
||||
| Key | Value |
|
||||
| ------ | ------ |
|
||||
| item_pgpdump_key:*item_path* | *key* |
|
||||
| | |
|
||||
| item_pgpdump_name:*item_path* | *name* |
|
||||
| | |
|
||||
| item_pgpdump_mail:*item_path* | *mail* |
|
||||
|
||||
#### Cryptocurrency
|
||||
|
||||
Supported cryptocurrency:
|
||||
- bitcoin
|
||||
|
||||
##### Hset:
|
||||
| Key | Field | Value |
|
||||
| ------ | ------ | ------ |
|
||||
| cryptocurrency_metadata_**cryptocurrency name**:**cryptocurrency address** | first_seen | **date** |
|
||||
| | last_seen | **date** |
|
||||
|
||||
##### set:
|
||||
| Key | Value |
|
||||
| ------ | ------ |
|
||||
| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** |
|
||||
|
||||
##### Hset date:
|
||||
| Key | Field | Value |
|
||||
| ------ | ------ |
|
||||
| cryptocurrency:**cryptocurrency name**:**date** | **cryptocurrency address** | **nb seen** |
|
||||
|
||||
##### zset:
|
||||
| Key | Field | Value |
|
||||
| ------ | ------ | ------ |
|
||||
| cryptocurrency_all:**cryptocurrency name** | **cryptocurrency address** | **nb seen** |
|
||||
|
||||
##### set:
|
||||
| Key | Value |
|
||||
| ------ | ------ |
|
||||
| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** |
|
||||
|
||||
|
||||
## DB9 - Crawler:
|
||||
|
||||
##### Hset:
|
||||
|
|
|
|||
|
|
@ -62,11 +62,12 @@ Features
|
|||
* Create events on [MISP](https://github.com/MISP/MISP) and cases on [The Hive](https://github.com/TheHive-Project/TheHive)
|
||||
* Automatic paste export at detection on [MISP](https://github.com/MISP/MISP) (events) and [The Hive](https://github.com/TheHive-Project/TheHive) (alerts) on selected tags
|
||||
* Extracted and decoded files can be searched by date range, type of file (mime-type) and encoding discovered
|
||||
* Graph relationships between decoded file (hashes)
|
||||
* Graph relationships between decoded file (hashes), similar PGP UIDs and addresses of cryptocurrencies
|
||||
* Tor hidden services crawler to crawl and parse output
|
||||
* Tor onion availability is monitored to detect up and down of hidden services
|
||||
* Browser hidden services are screenshot and integrated in the analysed output including a blurring screenshot interface (to avoid "burning the eyes" of the security analysis with specific content)
|
||||
* Tor hidden services is part of the standard framework, all the AIL modules are available to the crawled hidden services
|
||||
* Generic web crawler to trigger crawling on demand or at regular interval URL or Tor hidden services
|
||||
|
||||
|
||||
Installation
|
||||
|
|
|
|||
|
|
@ -145,15 +145,9 @@
|
|||
extra_args: --upgrade
|
||||
virtualenv: /opt/AIL-framework/AILENV
|
||||
|
||||
- name: Install pip requirements
|
||||
pip:
|
||||
requirements: /opt/AIL-framework/pip_packages_requirement.txt
|
||||
extra_args: --upgrade
|
||||
virtualenv: /opt/AIL-framework/AILENV
|
||||
|
||||
- name: Install pip3 requirements
|
||||
pip:
|
||||
requirements: /opt/AIL-framework/pip3_packages_requirement.txt
|
||||
requirements: /opt/AIL-framework/requirements.txt
|
||||
executable: pip3
|
||||
extra_args: --upgrade
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from pubsublogger import publisher
|
|||
|
||||
import re
|
||||
import time
|
||||
import redis
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
|
|
@ -44,6 +45,7 @@ def check_bc(bc):
|
|||
def search_key(content, message, paste):
|
||||
bitcoin_address = re.findall(regex_bitcoin_public_address, content)
|
||||
bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
|
||||
date = str(paste._get_p_date())
|
||||
validate_address = False
|
||||
key = False
|
||||
if(len(bitcoin_address) >0):
|
||||
|
|
@ -56,6 +58,8 @@ def search_key(content, message, paste):
|
|||
for private_key in bitcoin_private_key:
|
||||
print('Bitcoin private key found : {}'.format(private_key))
|
||||
key = True
|
||||
# build bitcoin correlation
|
||||
save_cryptocurrency_data('bitcoin', date, message, address)
|
||||
|
||||
if(validate_address):
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
|
|
@ -75,6 +79,31 @@ def search_key(content, message, paste):
|
|||
publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
|
||||
to_print, len(bitcoin_private_key),paste.p_rel_path))
|
||||
|
||||
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
||||
# create basic medata
|
||||
if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
||||
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
|
||||
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
else:
|
||||
last_seen = serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
|
||||
if not last_seen:
|
||||
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
else:
|
||||
if int(last_seen) < int(date):
|
||||
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
|
||||
# global set
|
||||
serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
|
||||
|
||||
# daily
|
||||
serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
|
||||
|
||||
# all type
|
||||
serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
|
||||
|
||||
# item_metadata
|
||||
serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
|
@ -84,6 +113,12 @@ if __name__ == "__main__":
|
|||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
serv_metadata = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Metadata", "host"),
|
||||
port=p.config.getint("ARDB_Metadata", "port"),
|
||||
db=p.config.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Run Keys module ")
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import time
|
|||
import subprocess
|
||||
import requests
|
||||
|
||||
from collections import deque
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
|
|
@ -43,25 +44,49 @@ def unpack_url(url):
|
|||
to_crawl = {}
|
||||
faup.decode(url)
|
||||
url_unpack = faup.get()
|
||||
to_crawl['domain'] = url_unpack['domain'].decode()
|
||||
# # FIXME: # TODO: remove me
|
||||
try:
|
||||
to_crawl['domain'] = url_unpack['domain'].decode()
|
||||
except:
|
||||
to_crawl['domain'] = url_unpack['domain']
|
||||
to_crawl['domain'] = to_crawl['domain'].lower()
|
||||
|
||||
|
||||
# force lower case domain/subdomain (rfc4343)
|
||||
# # FIXME: # TODO: remove me
|
||||
try:
|
||||
url_host = url_unpack['host'].decode()
|
||||
except:
|
||||
url_host = url_unpack['host']
|
||||
|
||||
new_url_host = url_host.lower()
|
||||
url_lower_case = url.replace(url_host, new_url_host, 1)
|
||||
|
||||
if url_unpack['scheme'] is None:
|
||||
to_crawl['scheme'] = 'http'
|
||||
url= 'http://{}'.format(url_unpack['url'].decode())
|
||||
url= 'http://{}'.format(url_lower_case)
|
||||
else:
|
||||
scheme = url_unpack['scheme'].decode()
|
||||
# # FIXME: # TODO: remove me
|
||||
try:
|
||||
scheme = url_unpack['scheme'].decode()
|
||||
except Exception as e:
|
||||
scheme = url_unpack['scheme']
|
||||
if scheme in default_proto_map:
|
||||
to_crawl['scheme'] = scheme
|
||||
url = url_unpack['url'].decode()
|
||||
url = url_lower_case
|
||||
else:
|
||||
redis_crawler.sadd('new_proto', '{} {}'.format(scheme, url_unpack['url'].decode()))
|
||||
redis_crawler.sadd('new_proto', '{} {}'.format(scheme, url_lower_case))
|
||||
to_crawl['scheme'] = 'http'
|
||||
url= 'http://{}'.format(url_unpack['url'].decode().replace(scheme, '', 1))
|
||||
url= 'http://{}'.format(url_lower_case.replace(scheme, '', 1))
|
||||
|
||||
if url_unpack['port'] is None:
|
||||
to_crawl['port'] = default_proto_map[to_crawl['scheme']]
|
||||
else:
|
||||
port = url_unpack['port'].decode()
|
||||
# # FIXME: # TODO: remove me
|
||||
try:
|
||||
port = url_unpack['port'].decode()
|
||||
except:
|
||||
port = url_unpack['port']
|
||||
# Verify port number #################### make function to verify/correct port number
|
||||
try:
|
||||
int(port)
|
||||
|
|
@ -80,12 +105,16 @@ def unpack_url(url):
|
|||
|
||||
to_crawl['url'] = url
|
||||
if to_crawl['port'] == 80:
|
||||
to_crawl['domain_url'] = '{}://{}'.format(to_crawl['scheme'], url_unpack['host'].decode())
|
||||
to_crawl['domain_url'] = '{}://{}'.format(to_crawl['scheme'], new_url_host)
|
||||
else:
|
||||
to_crawl['domain_url'] = '{}://{}:{}'.format(to_crawl['scheme'], url_unpack['host'].decode(), to_crawl['port'])
|
||||
to_crawl['domain_url'] = '{}://{}:{}'.format(to_crawl['scheme'], new_url_host, to_crawl['port'])
|
||||
|
||||
# # FIXME: # TODO: remove me
|
||||
try:
|
||||
to_crawl['tld'] = url_unpack['tld'].decode()
|
||||
except:
|
||||
to_crawl['tld'] = url_unpack['tld']
|
||||
|
||||
to_crawl['tld'] = url_unpack['tld'].decode()
|
||||
return to_crawl
|
||||
|
||||
# get url, paste and service_type to crawl
|
||||
|
|
@ -275,7 +304,7 @@ if __name__ == '__main__':
|
|||
#mode = sys.argv[1]
|
||||
splash_port = sys.argv[1]
|
||||
|
||||
rotation_mode = ['onion', 'regular']
|
||||
rotation_mode = deque(['onion', 'regular'])
|
||||
default_proto_map = {'http': 80, 'https': 443}
|
||||
######################################################## add ftp ???
|
||||
|
||||
|
|
@ -333,6 +362,7 @@ if __name__ == '__main__':
|
|||
|
||||
update_auto_crawler()
|
||||
|
||||
rotation_mode.rotate()
|
||||
to_crawl = get_elem_to_crawl(rotation_mode)
|
||||
if to_crawl:
|
||||
url_data = unpack_url(to_crawl['url'])
|
||||
|
|
|
|||
|
|
@ -124,6 +124,11 @@ if __name__ == "__main__":
|
|||
for url in sites:
|
||||
faup.decode(url)
|
||||
domain = faup.get()['domain']
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
domain = domain.decode()
|
||||
except:
|
||||
pass
|
||||
if domain in creds_sites.keys():
|
||||
creds_sites[domain] += 1
|
||||
else:
|
||||
|
|
@ -143,6 +148,11 @@ if __name__ == "__main__":
|
|||
maildomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", cred.lower())[0]
|
||||
faup.decode(maildomains)
|
||||
tld = faup.get()['tld']
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
tld = tld.decode()
|
||||
except:
|
||||
pass
|
||||
server_statistics.hincrby('credential_by_tld:'+date, tld, 1)
|
||||
else:
|
||||
publisher.info(to_print)
|
||||
|
|
|
|||
17
bin/Keys.py
17
bin/Keys.py
|
|
@ -25,13 +25,26 @@ from Helper import Process
|
|||
def search_key(paste):
|
||||
content = paste.get_p_content()
|
||||
find = False
|
||||
get_pgp_content = False
|
||||
if '-----BEGIN PGP MESSAGE-----' in content:
|
||||
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
|
||||
|
||||
msg = 'infoleak:automatic-detection="pgp-message";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
get_pgp_content = True
|
||||
find = True
|
||||
|
||||
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in content:
|
||||
msg = 'infoleak:automatic-detection="pgp-public-key-block";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
get_pgp_content = True
|
||||
|
||||
if '-----BEGIN PGP SIGNATURE-----' in content:
|
||||
msg = 'infoleak:automatic-detection="pgp-signature";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
get_pgp_content = True
|
||||
|
||||
|
||||
if '-----BEGIN CERTIFICATE-----' in content:
|
||||
publisher.warning('{} has a certificate message'.format(paste.p_name))
|
||||
|
||||
|
|
@ -108,6 +121,10 @@ def search_key(paste):
|
|||
p.populate_set_out(msg, 'Tags')
|
||||
find = True
|
||||
|
||||
# pgp content
|
||||
if get_pgp_content:
|
||||
p.populate_set_out(message, 'PgpDump')
|
||||
|
||||
if find :
|
||||
|
||||
#Send to duplicate
|
||||
|
|
|
|||
|
|
@ -187,6 +187,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x"
|
||||
|
|
|
|||
|
|
@ -29,8 +29,17 @@ def analyse(url, path):
|
|||
faup.decode(url)
|
||||
url_parsed = faup.get()
|
||||
pprint.pprint(url_parsed)
|
||||
resource_path = url_parsed['resource_path']
|
||||
query_string = url_parsed['query_string']
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
resource_path = url_parsed['resource_path'].encode()
|
||||
except:
|
||||
resource_path = url_parsed['resource_path']
|
||||
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
query_string = url_parsed['query_string'].encode()
|
||||
except:
|
||||
query_string = url_parsed['query_string']
|
||||
|
||||
result_path = {'sqli' : False}
|
||||
result_query = {'sqli' : False}
|
||||
|
|
@ -56,7 +65,11 @@ def analyse(url, path):
|
|||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
#statistics
|
||||
tld = url_parsed['tld']
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
tld = url_parsed['tld'].decode()
|
||||
except:
|
||||
tld = url_parsed['tld']
|
||||
if tld is not None:
|
||||
date = datetime.datetime.now().strftime("%Y%m")
|
||||
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
|
||||
|
|
|
|||
|
|
@ -95,6 +95,10 @@ if __name__ == "__main__":
|
|||
|
||||
faup.decode(mail)
|
||||
tld = faup.get()['tld']
|
||||
try:
|
||||
tld = tld.decode()
|
||||
except:
|
||||
pass
|
||||
server_statistics.hincrby('mail_by_tld:'+date, tld, MX_values[1][mail])
|
||||
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -224,7 +224,11 @@ if __name__ == "__main__":
|
|||
|
||||
faup.decode(url)
|
||||
url_unpack = faup.get()
|
||||
domain = url_unpack['domain'].decode()
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
domain = url_unpack['domain'].decode().lower()
|
||||
except Exception as e:
|
||||
domain = url_unpack['domain'].lower()
|
||||
|
||||
## TODO: blackilst by port ?
|
||||
# check blacklist
|
||||
|
|
@ -233,7 +237,7 @@ if __name__ == "__main__":
|
|||
|
||||
subdomain = re.findall(url_regex, url)
|
||||
if len(subdomain) > 0:
|
||||
subdomain = subdomain[0][4]
|
||||
subdomain = subdomain[0][4].lower()
|
||||
else:
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,242 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
PgpDum module
|
||||
|
||||
Extract ID from PGP Blocks
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
import redis
|
||||
import signal
|
||||
import datetime
|
||||
import subprocess
|
||||
|
||||
from pubsublogger import publisher
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from Helper import Process
|
||||
from packages import Paste
|
||||
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutException
|
||||
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
# save pgp message in directory, process one time each day
|
||||
def save_in_file(message, pgp_content):
|
||||
print('save in file')
|
||||
UUID = str(uuid.uuid4())
|
||||
file_name = os.path.join(pgp_dump_dir_to_process, UUID)
|
||||
with open(file_name, 'w') as f:
|
||||
f.write(pgp_content)
|
||||
r_serv_db.sadd('pgpdumb:uuid', '{};{}'.format(UUID, message))
|
||||
|
||||
def remove_html(item_content):
|
||||
if bool(BeautifulSoup(item_content, "html.parser").find()):
|
||||
soup = BeautifulSoup(item_content, 'html.parser')
|
||||
# kill all script and style elements
|
||||
for script in soup(["script", "style"]):
|
||||
script.extract() # remove
|
||||
|
||||
# get text
|
||||
text = soup.get_text()
|
||||
return text
|
||||
else:
|
||||
return item_content
|
||||
|
||||
def extract_all_id(message, item_content, regex=None, is_file=False):
|
||||
|
||||
if is_file:
|
||||
pgp_packet = get_pgp_packet_file(item_content)
|
||||
extract_id_from_output(pgp_packet)
|
||||
|
||||
else:
|
||||
# max execution time on regex
|
||||
signal.alarm(max_execution_time)
|
||||
try:
|
||||
pgp_extracted_block = re.findall(regex, item_content)
|
||||
except TimeoutException:
|
||||
pgp_extracted_block = []
|
||||
p.incr_module_timeout_statistic() # add encoder type
|
||||
print ("{0} processing timeout".format(paste.p_rel_path))
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
||||
for pgp_to_dump in pgp_extracted_block:
|
||||
pgp_packet = get_pgp_packet(message, pgp_to_dump)
|
||||
extract_id_from_output(pgp_packet)
|
||||
|
||||
def get_pgp_packet(message, save_path):
|
||||
save_path = '{}'.format(save_path)
|
||||
if len(save_path) > 131072:
|
||||
save_in_file(message, save_path)
|
||||
return ''
|
||||
else:
|
||||
process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE)
|
||||
process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE)
|
||||
process1.stdout.close()
|
||||
output = process2.communicate()[0].decode()
|
||||
return output
|
||||
|
||||
def get_pgp_packet_file(file):
|
||||
process1 = subprocess.Popen([ 'pgpdump', file], stdout=subprocess.PIPE)
|
||||
output = process1.communicate()[0].decode()
|
||||
return output
|
||||
|
||||
def extract_id_from_output(pgp_dump_outpout):
|
||||
all_user_id = set(re.findall(regex_user_id, pgp_dump_outpout))
|
||||
for user_id in all_user_id:
|
||||
# avoid key injection in user_id:
|
||||
pgp_dump_outpout.replace(user_id, '', 1)
|
||||
|
||||
user_id = user_id.replace(user_id_str, '', 1)
|
||||
mail = None
|
||||
if ' <' in user_id:
|
||||
name, mail = user_id.rsplit(' <', 1)
|
||||
mail = mail[:-1]
|
||||
set_name.add(name)
|
||||
set_mail.add(mail)
|
||||
else:
|
||||
name = user_id
|
||||
set_name.add(name)
|
||||
|
||||
all_key_id = set(re.findall(regex_key_id, pgp_dump_outpout))
|
||||
for key_id in all_key_id:
|
||||
key_id = key_id.replace(key_id_str, '', 1)
|
||||
set_key.add(key_id)
|
||||
|
||||
def save_pgp_data(type_pgp, date, item_path, data):
|
||||
# create basic medata
|
||||
if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)):
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date)
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
|
||||
else:
|
||||
last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen')
|
||||
if not last_seen:
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
|
||||
else:
|
||||
if int(last_seen) < int(date):
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
|
||||
|
||||
# global set
|
||||
serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path)
|
||||
|
||||
# daily
|
||||
serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1)
|
||||
|
||||
# all type
|
||||
serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1)
|
||||
|
||||
# item_metadata
|
||||
serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
#config_section = 'PgpDump'
|
||||
config_section = 'PgpDump'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
r_serv_db = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_DB", "host"),
|
||||
port=p.config.getint("ARDB_DB", "port"),
|
||||
db=p.config.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
serv_metadata = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Metadata", "host"),
|
||||
port=p.config.getint("ARDB_Metadata", "port"),
|
||||
db=p.config.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("PgpDump started")
|
||||
|
||||
# check/create pgpdump queue directory (used for huge pgp blocks)
|
||||
pgp_dump_dir_to_process = os.path.join(os.environ['AIL_HOME'], 'temp', 'pgpdump')
|
||||
if not os.path.isdir(pgp_dump_dir_to_process):
|
||||
os.makedirs(pgp_dump_dir_to_process)
|
||||
|
||||
user_id_str = 'User ID - '
|
||||
regex_user_id= '{}.+'.format(user_id_str)
|
||||
|
||||
key_id_str = 'Key ID - '
|
||||
regex_key_id = '{}.+'.format(key_id_str)
|
||||
regex_pgp_public_blocs = '-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
|
||||
regex_pgp_signature = '-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
|
||||
regex_pgp_message = '-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
|
||||
|
||||
re.compile(regex_user_id)
|
||||
re.compile(regex_key_id)
|
||||
re.compile(regex_pgp_public_blocs)
|
||||
re.compile(regex_pgp_signature)
|
||||
re.compile(regex_pgp_message)
|
||||
|
||||
max_execution_time = p.config.getint("PgpDump", "max_execution_time")
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
|
||||
is_file = False
|
||||
set_key = set()
|
||||
set_name = set()
|
||||
set_mail = set()
|
||||
|
||||
if r_serv_db.scard('pgpdumb:uuid') > 0:
|
||||
res = r_serv_db.spop('pgpdumb:uuid')
|
||||
file_to_process, message = res.split(';', 1)
|
||||
file_to_process = os.path.join(pgp_dump_dir_to_process, file_to_process)
|
||||
date = datetime.datetime.now().strftime("%Y/%m/%d")
|
||||
paste = Paste.Paste(message)
|
||||
date = str(paste._get_p_date())
|
||||
print(message)
|
||||
extract_all_id(message, file_to_process, is_file=True)
|
||||
os.remove(file_to_process)
|
||||
|
||||
else:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
|
||||
if message is None:
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
paste = Paste.Paste(message)
|
||||
|
||||
date = str(paste._get_p_date())
|
||||
content = paste.get_p_content()
|
||||
content = remove_html(content)
|
||||
|
||||
print(message)
|
||||
|
||||
extract_all_id(message, content, regex_pgp_public_blocs)
|
||||
extract_all_id(message, content, regex_pgp_signature)
|
||||
extract_all_id(message, content, regex_pgp_message)
|
||||
|
||||
for key_id in set_key:
|
||||
print(key_id)
|
||||
save_pgp_data('key', date, message, key_id)
|
||||
|
||||
for name_id in set_name:
|
||||
print(name_id)
|
||||
save_pgp_data('name', date, message, name_id)
|
||||
|
||||
for mail_id in set_mail:
|
||||
print(mail_id)
|
||||
save_pgp_data('mail', date, message, mail_id)
|
||||
|
|
@ -68,10 +68,20 @@ def analyse(url, path):
|
|||
result_query = 0
|
||||
|
||||
if resource_path is not None:
|
||||
result_path = is_sql_injection(resource_path.decode('utf8'))
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
resource_path = resource_path.decode()
|
||||
except:
|
||||
pass
|
||||
result_path = is_sql_injection(resource_path)
|
||||
|
||||
if query_string is not None:
|
||||
result_query = is_sql_injection(query_string.decode('utf8'))
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
query_string = query_string.decode()
|
||||
except:
|
||||
pass
|
||||
result_query = is_sql_injection(query_string)
|
||||
|
||||
if (result_path > 0) or (result_query > 0):
|
||||
paste = Paste.Paste(path)
|
||||
|
|
@ -89,6 +99,11 @@ def analyse(url, path):
|
|||
#statistics
|
||||
tld = url_parsed['tld']
|
||||
if tld is not None:
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
tld = tld.decode()
|
||||
except:
|
||||
pass
|
||||
date = datetime.datetime.now().strftime("%Y%m")
|
||||
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
|
||||
|
||||
|
|
|
|||
|
|
@ -145,10 +145,14 @@ def get_git_upper_tags_remote(current_tag, is_fork):
|
|||
list_upper_tags = []
|
||||
if list_all_tags[-1][1:] == current_tag:
|
||||
list_upper_tags.append( (list_all_tags[-1], None) )
|
||||
# force update order
|
||||
list_upper_tags.sort()
|
||||
return list_upper_tags
|
||||
for tag in list_all_tags:
|
||||
if float(tag[1:]) >= float(current_tag):
|
||||
list_upper_tags.append( (tag, None) )
|
||||
# force update order
|
||||
list_upper_tags.sort()
|
||||
return list_upper_tags
|
||||
else:
|
||||
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
|
||||
|
|
@ -165,6 +169,8 @@ def get_git_upper_tags_remote(current_tag, is_fork):
|
|||
list_upper_tags = []
|
||||
if last_tag[1:] == current_tag:
|
||||
list_upper_tags.append( (last_tag, last_commit) )
|
||||
# force update order
|
||||
list_upper_tags.sort()
|
||||
return list_upper_tags
|
||||
else:
|
||||
for mess_tag in list_all_tags:
|
||||
|
|
@ -177,6 +183,8 @@ def get_git_upper_tags_remote(current_tag, is_fork):
|
|||
# add last commit
|
||||
if last_tag not in list_upper_tags[-1][0]:
|
||||
list_upper_tags.append( (last_tag, last_commit) )
|
||||
# force update order
|
||||
list_upper_tags.sort()
|
||||
return list_upper_tags
|
||||
|
||||
else:
|
||||
|
|
|
|||
16
bin/Web.py
16
bin/Web.py
|
|
@ -94,18 +94,22 @@ if __name__ == "__main__":
|
|||
faup.decode(url)
|
||||
domain = faup.get_domain()
|
||||
subdomain = faup.get_subdomain()
|
||||
f1 = None
|
||||
|
||||
publisher.debug('{} Published'.format(url))
|
||||
|
||||
if f1 == "onion":
|
||||
print(domain)
|
||||
|
||||
if subdomain is not None:
|
||||
subdomain = subdomain.decode('utf8')
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
subdomain = subdomain.decode()
|
||||
except:
|
||||
pass
|
||||
|
||||
if domain is not None:
|
||||
domain = domain.decode('utf8')
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
domain = domain.decode()
|
||||
except:
|
||||
pass
|
||||
domains_list.append(domain)
|
||||
|
||||
hostl = avoidNone(subdomain) + avoidNone(domain)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import base64
|
||||
import uuid
|
||||
import datetime
|
||||
import base64
|
||||
import redis
|
||||
import json
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
config_section = 'Global'
|
||||
p = Process(config_section)
|
||||
|
||||
r_tags = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Tags", "host"),
|
||||
port=p.config.getint("ARDB_Tags", "port"),
|
||||
db=p.config.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
tag = 'infoleak:automatic-detection="bitcoin-address"'
|
||||
|
||||
# get tag first/last seen
|
||||
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
|
||||
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
|
||||
l_dates = substract_date(first_seen, last_seen)
|
||||
|
||||
# get all tagged items
|
||||
for date in l_dates:
|
||||
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
|
||||
|
||||
for item in daily_tagged_items:
|
||||
p.populate_set_out(item)
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import base64
|
||||
import uuid
|
||||
import datetime
|
||||
import base64
|
||||
import redis
|
||||
import json
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
config_section = 'Keys'
|
||||
p = Process(config_section)
|
||||
|
||||
r_tags = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Tags", "host"),
|
||||
port=p.config.getint("ARDB_Tags", "port"),
|
||||
db=p.config.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
tag = 'infoleak:automatic-detection="pgp-message"'
|
||||
|
||||
# get tag first/last seen
|
||||
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
|
||||
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
|
||||
l_dates = substract_date(first_seen, last_seen)
|
||||
|
||||
# get all tagged items
|
||||
for date in l_dates:
|
||||
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
|
||||
|
||||
for item in daily_tagged_items:
|
||||
p.populate_set_out(item, 'PgpDump')
|
||||
|
|
@ -125,7 +125,15 @@ class Paste(object):
|
|||
|
||||
"""
|
||||
|
||||
paste = self.cache.get(self.p_path)
|
||||
try:
|
||||
paste = self.cache.get(self.p_path)
|
||||
except UnicodeDecodeError:
|
||||
paste = None
|
||||
except Exception as e:
|
||||
print("ERROR in: " + self.p_path)
|
||||
print(e)
|
||||
paste = None
|
||||
|
||||
if paste is None:
|
||||
try:
|
||||
with gzip.open(self.p_path, 'r') as f:
|
||||
|
|
|
|||
|
|
@ -71,6 +71,9 @@ max_execution_time = 90
|
|||
[Onion]
|
||||
max_execution_time = 180
|
||||
|
||||
[PgpDump]
|
||||
max_execution_time = 60
|
||||
|
||||
[Base64]
|
||||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
|
|
|||
|
|
@ -118,6 +118,10 @@ publish = Redis_Duplicate,Redis_Tags
|
|||
|
||||
[Keys]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_PgpDump,Redis_Tags
|
||||
|
||||
[PgpDump]
|
||||
subscribe = Redis_PgpDump
|
||||
publish = Redis_Duplicate,Redis_Tags
|
||||
|
||||
[ApiKey]
|
||||
|
|
|
|||
|
|
@ -48,15 +48,16 @@ class TorSplashCrawler():
|
|||
'DEPTH_LIMIT': crawler_options['depth_limit']
|
||||
})
|
||||
|
||||
def crawl(self, type, crawler_options, date, url, domain, port, original_item):
|
||||
self.process.crawl(self.crawler, type=type, crawler_options=crawler_options, date=date, url=url, domain=domain, port=port, original_item=original_item)
|
||||
def crawl(self, type, crawler_options, date, requested_mode, url, domain, port, original_item):
|
||||
self.process.crawl(self.crawler, type=type, crawler_options=crawler_options, date=date, requested_mode=requested_mode, url=url, domain=domain, port=port, original_item=original_item)
|
||||
self.process.start()
|
||||
|
||||
class TorSplashSpider(Spider):
|
||||
name = 'TorSplashSpider'
|
||||
|
||||
def __init__(self, type, crawler_options, date, url, domain, port, original_item, *args, **kwargs):
|
||||
def __init__(self, type, crawler_options, date, requested_mode, url, domain, port, original_item, *args, **kwargs):
|
||||
self.type = type
|
||||
self.requested_mode = requested_mode
|
||||
self.original_item = original_item
|
||||
self.root_key = None
|
||||
self.start_urls = url
|
||||
|
|
@ -183,7 +184,7 @@ class TorSplashCrawler():
|
|||
if 'png' in response.data:
|
||||
size_screenshot = (len(response.data['png'])*3) /4
|
||||
|
||||
if size_screenshot < 5000000: #bytes
|
||||
if size_screenshot < 5000000 or self.requested_mode: #bytes or manual/auto
|
||||
image_content = base64.standard_b64decode(response.data['png'].encode())
|
||||
hash = sha256(image_content).hexdigest()
|
||||
img_dir_path = os.path.join(hash[0:2], hash[2:4], hash[4:6], hash[6:8], hash[8:10], hash[10:12])
|
||||
|
|
|
|||
|
|
@ -43,8 +43,9 @@ if __name__ == '__main__':
|
|||
original_item = crawler_json['item']
|
||||
crawler_options = crawler_json['crawler_options']
|
||||
date = crawler_json['date']
|
||||
requested_mode = crawler_json['requested']
|
||||
|
||||
redis_cache.delete('crawler_request:{}'.format(uuid))
|
||||
|
||||
crawler = TorSplashCrawler(splash_url, crawler_options)
|
||||
crawler.crawl(service_type, crawler_options, date, url, domain, port, original_item)
|
||||
crawler.crawl(service_type, crawler_options, date, requested_mode, url, domain, port, original_item)
|
||||
|
|
|
|||
|
|
@ -67,6 +67,14 @@ sudo ldconfig
|
|||
popd
|
||||
popd
|
||||
|
||||
# pgpdump
|
||||
test ! -d pgpdump && git clone https://github.com/kazu-yamamoto/pgpdump.git
|
||||
pushd pgpdump/
|
||||
./configure
|
||||
make
|
||||
sudo make install
|
||||
popd
|
||||
|
||||
# ARDB #
|
||||
test ! -d ardb/ && git clone https://github.com/yinqiwen/ardb.git
|
||||
pushd ardb/
|
||||
|
|
@ -99,7 +107,7 @@ mkdir -p $AIL_HOME/PASTES
|
|||
|
||||
pip3 install -U pip
|
||||
pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client'
|
||||
pip3 install -U -r pip3_packages_requirement.txt
|
||||
pip3 install -U -r requirements.txt
|
||||
|
||||
# Pyfaup
|
||||
pushd faup/src/lib/bindings/python/
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ DomainClassifier
|
|||
#Indexer requirements
|
||||
whoosh
|
||||
|
||||
beautifulsoup4
|
||||
|
||||
ipaddress
|
||||
pycountry
|
||||
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_DB", "host"),
|
||||
port=cfg.getint("ARDB_DB", "port"),
|
||||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', 'v1.7')
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:update_date_v1.7', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k &
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update DomainClassifier"$DEFAULT
|
||||
echo ""
|
||||
|
||||
cd $AIL_HOME
|
||||
git clone https://github.com/kazu-yamamoto/pgpdump.git
|
||||
cd pgpdump
|
||||
./configure
|
||||
make
|
||||
sudo make install
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update requirement"$DEFAULT
|
||||
echo ""
|
||||
pip3 install beautifulsoup4
|
||||
|
||||
bash ${AIL_BIN}LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v1.7/Update.py &
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k &
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t &
|
||||
wait
|
||||
|
||||
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
|
|
@ -116,10 +116,6 @@ except IOError:
|
|||
f = open('templates/ignored_modules.txt', 'w')
|
||||
f.close()
|
||||
|
||||
activate_crawler = cfg.get("Crawler", "activate_crawler")
|
||||
if activate_crawler != 'True':
|
||||
toIgnoreModule.add('hiddenServices')
|
||||
|
||||
# Dynamically import routes and functions from modules
|
||||
# Also, prepare header.html
|
||||
to_add_to_header_dico = {}
|
||||
|
|
|
|||
|
|
@ -173,6 +173,8 @@ REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git'
|
|||
|
||||
max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
|
||||
|
||||
crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
|
||||
|
||||
# VT
|
||||
try:
|
||||
from virusTotalKEYS import vt_key
|
||||
|
|
|
|||
|
|
@ -297,6 +297,13 @@ def submit():
|
|||
ltagsgalaxies = request.form['tags_galaxies']
|
||||
paste_content = request.form['paste_content']
|
||||
|
||||
is_file = False
|
||||
if 'file' in request.files:
|
||||
file = request.files['file']
|
||||
if file:
|
||||
if file.filename:
|
||||
is_file = True
|
||||
|
||||
submitted_tag = 'infoleak:submission="manual"'
|
||||
|
||||
#active taxonomies
|
||||
|
|
@ -307,6 +314,7 @@ def submit():
|
|||
if ltags or ltagsgalaxies:
|
||||
if not addTagsVerification(ltags, ltagsgalaxies):
|
||||
content = 'INVALID TAGS'
|
||||
print(content)
|
||||
return content, 400
|
||||
|
||||
# add submitted tags
|
||||
|
|
@ -315,9 +323,7 @@ def submit():
|
|||
else:
|
||||
ltags = submitted_tag
|
||||
|
||||
if 'file' in request.files:
|
||||
|
||||
file = request.files['file']
|
||||
if is_file:
|
||||
if file:
|
||||
|
||||
if file and allowed_file(file.filename):
|
||||
|
|
@ -357,6 +363,7 @@ def submit():
|
|||
|
||||
else:
|
||||
content = 'wrong file type, allowed_extensions: sh, pdf, zip, gz, tar.gz or remove the extension'
|
||||
print(content)
|
||||
return content, 400
|
||||
|
||||
|
||||
|
|
@ -379,9 +386,11 @@ def submit():
|
|||
|
||||
else:
|
||||
content = 'size error'
|
||||
print(content)
|
||||
return content, 400
|
||||
|
||||
content = 'submit aborded'
|
||||
print(content)
|
||||
return content, 400
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ from Date import Date
|
|||
from io import BytesIO
|
||||
import zipfile
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
import requests
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
|
||||
from flask_login import login_required
|
||||
|
|
@ -30,6 +32,10 @@ PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
|||
|
||||
hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
|
||||
|
||||
## TODO: put me in option
|
||||
all_cryptocurrency = ['bitcoin', 'monero']
|
||||
all_pgpdump = ['key', 'name', 'mail']
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def get_date_range(num_day):
|
||||
|
|
@ -67,13 +73,13 @@ def get_file_icon(estimated_type):
|
|||
if file_type == 'application':
|
||||
file_icon = 'fa-file '
|
||||
elif file_type == 'audio':
|
||||
file_icon = 'fa-file-video-o '
|
||||
file_icon = 'fa-file-audio '
|
||||
elif file_type == 'image':
|
||||
file_icon = 'fa-file-image-o'
|
||||
file_icon = 'fa-file-image'
|
||||
elif file_type == 'text':
|
||||
file_icon = 'fa-file-text-o'
|
||||
file_icon = 'fa-file-alt'
|
||||
else:
|
||||
file_icon = 'fa-file-o'
|
||||
file_icon = 'fa-sticky-note'
|
||||
|
||||
return file_icon
|
||||
|
||||
|
|
@ -89,13 +95,384 @@ def get_file_icon_text(estimated_type):
|
|||
elif file_type == 'text':
|
||||
file_icon_text = '\uf15c'
|
||||
else:
|
||||
file_icon_text = '\uf15b'
|
||||
file_icon_text = '\uf249'
|
||||
|
||||
return file_icon_text
|
||||
|
||||
def get_icon(correlation_type, type_id):
|
||||
icon_text = 'fas fa-sticky-note'
|
||||
if correlation_type == 'pgpdump':
|
||||
# set type_id icon
|
||||
if type_id == 'key':
|
||||
icon_text = 'fas fa-key'
|
||||
elif type_id == 'name':
|
||||
icon_text = 'fas fa-user-tag'
|
||||
elif type_id == 'mail':
|
||||
icon_text = 'fas fa-at'
|
||||
else:
|
||||
icon_text = 'times'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
if type_id == 'bitcoin':
|
||||
icon_text = 'fab fa-btc'
|
||||
elif type_id == 'monero':
|
||||
icon_text = 'fab fa-monero'
|
||||
elif type_id == 'ethereum':
|
||||
icon_text = 'fab fa-ethereum'
|
||||
else:
|
||||
icon_text = 'fas fa-coins'
|
||||
return icon_text
|
||||
|
||||
def get_icon_text(correlation_type, type_id):
|
||||
icon_text = '\uf249'
|
||||
if correlation_type == 'pgpdump':
|
||||
if type_id == 'key':
|
||||
icon_text = '\uf084'
|
||||
elif type_id == 'name':
|
||||
icon_text = '\uf507'
|
||||
elif type_id == 'mail':
|
||||
icon_text = '\uf1fa'
|
||||
else:
|
||||
icon_text = 'times'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
if type_id == 'bitcoin':
|
||||
icon_text = '\uf15a'
|
||||
elif type_id == 'monero':
|
||||
icon_text = '\uf3d0'
|
||||
elif type_id == 'ethereum':
|
||||
icon_text = '\uf42e'
|
||||
else:
|
||||
icon_text = '\uf51e'
|
||||
return icon_text
|
||||
|
||||
def get_all_types_id(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
return all_pgpdump
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
return all_cryptocurrency
|
||||
else:
|
||||
return []
|
||||
|
||||
def is_valid_type_id(correlation_type, type_id):
|
||||
all_type_id = get_all_types_id(correlation_type)
|
||||
if type_id in all_type_id:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def get_key_id_metadata(correlation_type, type_id, key_id):
|
||||
key_id_metadata = {}
|
||||
if r_serv_metadata.exists('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id)):
|
||||
key_id_metadata['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id), 'first_seen')
|
||||
key_id_metadata['first_seen'] = '{}/{}/{}'.format(key_id_metadata['first_seen'][0:4], key_id_metadata['first_seen'][4:6], key_id_metadata['first_seen'][6:8])
|
||||
key_id_metadata['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id), 'last_seen')
|
||||
key_id_metadata['last_seen'] = '{}/{}/{}'.format(key_id_metadata['last_seen'][0:4], key_id_metadata['last_seen'][4:6], key_id_metadata['last_seen'][6:8])
|
||||
key_id_metadata['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(correlation_type, type_id, key_id))
|
||||
return key_id_metadata
|
||||
|
||||
def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id):
|
||||
sparklines_value = []
|
||||
for date_day in date_range_sparkline:
|
||||
nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date_day), key_id)
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
sparklines_value.append(int(nb_seen_this_day))
|
||||
return sparklines_value
|
||||
|
||||
def get_all_keys_id_from_item(correlation_type, item_path):
|
||||
all_keys_id_dump = set()
|
||||
if item_path is not None:
|
||||
for type_id in get_all_types_id(correlation_type):
|
||||
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_type, type_id, item_path))
|
||||
for key_id in res:
|
||||
all_keys_id_dump.add( (key_id, type_id) )
|
||||
return all_keys_id_dump
|
||||
|
||||
def one():
|
||||
return 1
|
||||
|
||||
'''
|
||||
def decode_base58(bc, length):
|
||||
n = 0
|
||||
for char in bc:
|
||||
n = n * 58 + digits58.index(char)
|
||||
return n.to_bytes(length, 'big')
|
||||
|
||||
def check_bc(bc):
|
||||
try:
|
||||
bcbytes = decode_base58(bc, 25)
|
||||
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
|
||||
except Exception:
|
||||
return False
|
||||
'''
|
||||
|
||||
def get_correlation_type_search_endpoint(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
endpoint = 'hashDecoded.all_pgpdump_search'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
endpoint = 'hashDecoded.all_cryptocurrency_search'
|
||||
else:
|
||||
endpoint = 'hashDecoded.hashDecoded_page'
|
||||
return endpoint
|
||||
|
||||
def get_correlation_type_page_endpoint(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
endpoint = 'hashDecoded.pgpdump_page'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
endpoint = 'hashDecoded.cryptocurrency_page'
|
||||
else:
|
||||
endpoint = 'hashDecoded.hashDecoded_page'
|
||||
return endpoint
|
||||
|
||||
def get_show_key_id_endpoint(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
endpoint = 'hashDecoded.show_pgpdump'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
endpoint = 'hashDecoded.show_cryptocurrency'
|
||||
else:
|
||||
endpoint = 'hashDecoded.hashDecoded_page'
|
||||
return endpoint
|
||||
|
||||
def get_range_type_json_endpoint(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
endpoint = 'hashDecoded.pgpdump_range_type_json'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
endpoint = 'hashDecoded.cryptocurrency_range_type_json'
|
||||
else:
|
||||
endpoint = 'hashDecoded.hashDecoded_page'
|
||||
return endpoint
|
||||
|
||||
def get_graph_node_json_endpoint(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
endpoint = 'hashDecoded.pgpdump_graph_node_json'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
endpoint = 'hashDecoded.cryptocurrency_graph_node_json'
|
||||
else:
|
||||
endpoint = 'hashDecoded.hashDecoded_page'
|
||||
return endpoint
|
||||
|
||||
def get_graph_line_json_endpoint(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
endpoint = 'hashDecoded.pgpdump_graph_line_json'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
endpoint = 'hashDecoded.cryptocurrency_graph_line_json'
|
||||
else:
|
||||
endpoint = 'hashDecoded.hashDecoded_page'
|
||||
return endpoint
|
||||
|
||||
def get_font_family(correlation_type):
|
||||
if correlation_type == 'pgpdump':
|
||||
font = 'fa'
|
||||
elif correlation_type == 'cryptocurrency':
|
||||
font = 'fab'
|
||||
else:
|
||||
font = 'fa'
|
||||
return font
|
||||
|
||||
############ CORE CORRELATION ############
|
||||
|
||||
def main_correlation_page(correlation_type, type_id, date_from, date_to, show_decoded_files):
|
||||
|
||||
if type_id == 'All types':
|
||||
type_id = None
|
||||
|
||||
# verify type input
|
||||
if type_id is not None:
|
||||
#retrieve char
|
||||
type_id = type_id.replace(' ', '')
|
||||
if not is_valid_type_id(correlation_type, type_id):
|
||||
type_id = None
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
try:
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
except:
|
||||
pass
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
|
||||
date_to = date_from
|
||||
|
||||
else:
|
||||
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
||||
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
||||
|
||||
# display day type bar chart
|
||||
if len(date_range) == 1 and type is None:
|
||||
daily_type_chart = True
|
||||
daily_date = date_range[0]
|
||||
else:
|
||||
daily_type_chart = False
|
||||
daily_date = None
|
||||
|
||||
if type_id is None:
|
||||
all_type_id = get_all_types_id(correlation_type)
|
||||
else:
|
||||
all_type_id = type_id
|
||||
|
||||
l_keys_id_dump = set()
|
||||
if show_decoded_files:
|
||||
for date in date_range:
|
||||
if isinstance(all_type_id, str):
|
||||
l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, all_type_id, date))
|
||||
if l_dump:
|
||||
for dump in l_dump:
|
||||
l_keys_id_dump.add( (dump, all_type_id) )
|
||||
else:
|
||||
for typ_id in all_type_id:
|
||||
l_dump = r_serv_metadata.hkeys('{}:{}:{}'.format(correlation_type, typ_id, date))
|
||||
if l_dump:
|
||||
for dump in l_dump:
|
||||
l_keys_id_dump.add( (dump, typ_id) )
|
||||
|
||||
|
||||
num_day_sparkline = 6
|
||||
date_range_sparkline = get_date_range(num_day_sparkline)
|
||||
|
||||
sparkline_id = 0
|
||||
keys_id_metadata = {}
|
||||
for dump_res in l_keys_id_dump:
|
||||
new_key_id, typ_id = dump_res
|
||||
|
||||
keys_id_metadata[new_key_id] = get_key_id_metadata(correlation_type, typ_id, new_key_id)
|
||||
|
||||
if keys_id_metadata[new_key_id]:
|
||||
keys_id_metadata[new_key_id]['type_id'] = typ_id
|
||||
keys_id_metadata[new_key_id]['type_icon'] = get_icon(correlation_type, typ_id)
|
||||
|
||||
keys_id_metadata[new_key_id]['sparklines_data'] = list_sparkline_type_id_values(date_range_sparkline, correlation_type, typ_id, new_key_id)
|
||||
keys_id_metadata[new_key_id]['sparklines_id'] = sparkline_id
|
||||
sparkline_id += 1
|
||||
|
||||
l_type = get_all_types_id(correlation_type)
|
||||
|
||||
return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata,
|
||||
correlation_type=correlation_type,
|
||||
correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type),
|
||||
correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type),
|
||||
show_key_id_endpoint=get_show_key_id_endpoint(correlation_type),
|
||||
range_type_json_endpoint=get_range_type_json_endpoint(correlation_type),
|
||||
l_type=l_type, type_id=type_id,
|
||||
daily_type_chart=daily_type_chart, daily_date=daily_date,
|
||||
date_from=date_from, date_to=date_to,
|
||||
show_decoded_files=show_decoded_files)
|
||||
|
||||
def show_correlation(correlation_type, type_id, key_id):
|
||||
if is_valid_type_id(correlation_type, type_id):
|
||||
key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id)
|
||||
if key_id_metadata:
|
||||
|
||||
num_day_sparkline = 6
|
||||
date_range_sparkline = get_date_range(num_day_sparkline)
|
||||
|
||||
sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id)
|
||||
return render_template('showCorrelation.html', key_id=key_id, type_id=type_id,
|
||||
correlation_type=correlation_type,
|
||||
graph_node_endpoint=get_graph_node_json_endpoint(correlation_type),
|
||||
graph_line_endpoint=get_graph_line_json_endpoint(correlation_type),
|
||||
font_family=get_font_family(correlation_type),
|
||||
key_id_metadata=key_id_metadata,
|
||||
type_icon=get_icon(correlation_type, type_id),
|
||||
sparkline_values=sparkline_values)
|
||||
else:
|
||||
return '404'
|
||||
else:
|
||||
return 'error'
|
||||
|
||||
def correlation_type_range_type_json(correlation_type, date_from, date_to):
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
|
||||
range_type = []
|
||||
all_types_id = get_all_types_id(correlation_type)
|
||||
|
||||
# one day
|
||||
if len(date_range) == 1:
|
||||
for type_id in all_types_id:
|
||||
day_type = {}
|
||||
# init 0
|
||||
for typ_id in all_types_id:
|
||||
day_type[typ_id] = 0
|
||||
day_type['date'] = type_id
|
||||
num_day_type_id = 0
|
||||
all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date_range[0]))
|
||||
if all_keys:
|
||||
for val in all_keys:
|
||||
num_day_type_id += int(val)
|
||||
day_type[type_id]= num_day_type_id
|
||||
|
||||
#if day_type[type_id] != 0:
|
||||
range_type.append(day_type)
|
||||
|
||||
else:
|
||||
# display type_id
|
||||
for date in date_range:
|
||||
day_type = {}
|
||||
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for type_id in all_types_id:
|
||||
num_day_type_id = 0
|
||||
all_keys = r_serv_metadata.hvals('{}:{}:{}'.format(correlation_type, type_id, date))
|
||||
if all_keys:
|
||||
for val in all_keys:
|
||||
num_day_type_id += int(val)
|
||||
day_type[type_id]= num_day_type_id
|
||||
range_type.append(day_type)
|
||||
|
||||
return jsonify(range_type)
|
||||
|
||||
def correlation_graph_node_json(correlation_type, type_id, key_id):
|
||||
if key_id is not None and is_valid_type_id(correlation_type, type_id):
|
||||
|
||||
nodes_set_dump = set()
|
||||
nodes_set_paste = set()
|
||||
links_set = set()
|
||||
|
||||
key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id)
|
||||
|
||||
nodes_set_dump.add((key_id, 1, type_id, key_id_metadata['first_seen'], key_id_metadata['last_seen'], key_id_metadata['nb_seen']))
|
||||
|
||||
#get related paste
|
||||
l_pastes = r_serv_metadata.smembers('set_{}_{}:{}'.format(correlation_type, type_id, key_id))
|
||||
for paste in l_pastes:
|
||||
nodes_set_paste.add((paste, 2))
|
||||
links_set.add((key_id, paste))
|
||||
|
||||
for key_id_with_type_id in get_all_keys_id_from_item(correlation_type, paste):
|
||||
new_key_id, typ_id = key_id_with_type_id
|
||||
if new_key_id != key_id:
|
||||
|
||||
key_id_metadata = get_key_id_metadata(correlation_type, typ_id, new_key_id)
|
||||
|
||||
nodes_set_dump.add((new_key_id, 3, typ_id, key_id_metadata['first_seen'], key_id_metadata['last_seen'], key_id_metadata['nb_seen']))
|
||||
links_set.add((new_key_id, paste))
|
||||
|
||||
nodes = []
|
||||
for node in nodes_set_dump:
|
||||
nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_icon_text(correlation_type, node[2]),"url": url_for(get_show_key_id_endpoint(correlation_type), type_id=node[2], key_id=node[0]), 'hash': True})
|
||||
for node in nodes_set_paste:
|
||||
nodes.append({"id": node[0], "group": node[1],"url": url_for('showsavedpastes.showsavedpaste', paste=node[0]), 'hash': False})
|
||||
links = []
|
||||
for link in links_set:
|
||||
links.append({"source": link[0], "target": link[1]})
|
||||
json = {"nodes": nodes, "links": links}
|
||||
return jsonify(json)
|
||||
|
||||
else:
|
||||
return jsonify({})
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
|
||||
@login_required
|
||||
|
|
@ -107,7 +484,6 @@ def all_hash_search():
|
|||
show_decoded_files = request.form.get('show_decoded_files')
|
||||
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files))
|
||||
|
||||
|
||||
@hashDecoded.route("/hashDecoded/", methods=['GET'])
|
||||
@login_required
|
||||
def hashDecoded_page():
|
||||
|
|
@ -723,5 +1099,164 @@ def update_vt_result():
|
|||
# TODO FIXME make json response
|
||||
return jsonify()
|
||||
|
||||
############################ PGPDump ############################
|
||||
|
||||
@hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR
|
||||
def pgp_by_type_json():
|
||||
type_id = request.args.get('type_id')
|
||||
date_from = request.args.get('date_from')
|
||||
|
||||
if date_from is None:
|
||||
date_from = datetime.date.today().strftime("%Y%m%d")
|
||||
|
||||
#retrieve + char
|
||||
type_id = type_id.replace(' ', '+')
|
||||
default = False
|
||||
|
||||
if type_id is None:
|
||||
default = True
|
||||
all_type = ['key', 'name', 'mail']
|
||||
else:
|
||||
all_type = [ type_id ]
|
||||
|
||||
num_day_type = 30
|
||||
date_range = get_date_range(num_day_type)
|
||||
|
||||
#verify input
|
||||
if verify_pgp_type_id(type_id) or default:
|
||||
|
||||
type_value = []
|
||||
|
||||
range_decoder = []
|
||||
for date in date_range:
|
||||
day_type_id = {}
|
||||
day_type_id['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for type_pgp in all_type:
|
||||
all_vals_key = r_serv_metadata.hvals('pgp:{}:date'.format(type_id, date))
|
||||
num_day_type_id = 0
|
||||
if all_vals_key is not None:
|
||||
for val_key in all_vals_key:
|
||||
num_day_type_id += int(val_key)
|
||||
day_type_id[type_pgp]= num_day_type_id
|
||||
range_decoder.append(day_type_id)
|
||||
|
||||
return jsonify(range_decoder)
|
||||
else:
|
||||
return jsonify()
|
||||
|
||||
############################ Correlation ############################
|
||||
@hashDecoded.route("/correlation/pgpdump", methods=['GET'])
|
||||
def pgpdump_page():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
type_id = request.args.get('type_id')
|
||||
|
||||
show_decoded_files = request.args.get('show_decoded_files')
|
||||
res = main_correlation_page('pgpdump', type_id, date_from, date_to, show_decoded_files)
|
||||
return res
|
||||
|
||||
@hashDecoded.route("/correlation/cryptocurrency", methods=['GET'])
|
||||
def cryptocurrency_page():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
type_id = request.args.get('type_id')
|
||||
|
||||
show_decoded_files = request.args.get('show_decoded_files')
|
||||
res = main_correlation_page('cryptocurrency', type_id, date_from, date_to, show_decoded_files)
|
||||
return res
|
||||
|
||||
@hashDecoded.route("/correlation/all_pgpdump_search", methods=['POST'])
|
||||
def all_pgpdump_search():
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
type_id = request.form.get('type')
|
||||
show_decoded_files = request.form.get('show_decoded_files')
|
||||
return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
|
||||
|
||||
@hashDecoded.route("/correlation/all_cryptocurrency_search", methods=['POST'])
|
||||
def all_cryptocurrency_search():
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
type_id = request.form.get('type')
|
||||
show_decoded_files = request.form.get('show_decoded_files')
|
||||
return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
|
||||
|
||||
@hashDecoded.route('/correlation/show_pgpdump')
|
||||
def show_pgpdump():
|
||||
type_id = request.args.get('type_id')
|
||||
key_id = request.args.get('key_id')
|
||||
return show_correlation('pgpdump', type_id, key_id)
|
||||
|
||||
|
||||
@hashDecoded.route('/correlation/show_cryptocurrency')
|
||||
def show_cryptocurrency():
|
||||
type_id = request.args.get('type_id')
|
||||
key_id = request.args.get('key_id')
|
||||
return show_correlation('cryptocurrency', type_id, key_id)
|
||||
|
||||
@hashDecoded.route('/correlation/cryptocurrency_range_type_json')
|
||||
def cryptocurrency_range_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
return correlation_type_range_type_json('cryptocurrency', date_from, date_to)
|
||||
|
||||
@hashDecoded.route('/correlation/pgpdump_range_type_json')
|
||||
def pgpdump_range_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
return correlation_type_range_type_json('pgpdump', date_from, date_to)
|
||||
|
||||
@hashDecoded.route('/correlation/pgpdump_graph_node_json')
|
||||
def pgpdump_graph_node_json():
|
||||
type_id = request.args.get('type_id')
|
||||
key_id = request.args.get('key_id')
|
||||
return correlation_graph_node_json('pgpdump', type_id, key_id)
|
||||
|
||||
@hashDecoded.route('/correlation/cryptocurrency_graph_node_json')
|
||||
def cryptocurrency_graph_node_json():
|
||||
type_id = request.args.get('type_id')
|
||||
key_id = request.args.get('key_id')
|
||||
return correlation_graph_node_json('cryptocurrency', type_id, key_id)
|
||||
|
||||
@hashDecoded.route('/correlation/pgpdump_graph_line_json')
|
||||
def pgpdump_graph_line_json():
|
||||
type_id = request.args.get('type_id')
|
||||
key_id = request.args.get('key_id')
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
return correlation_graph_line_json('pgpdump', type_id, key_id, date_from, date_to)
|
||||
|
||||
def correlation_graph_line_json(correlation_type, type_id, key_id, date_from, date_to):
|
||||
# verify input
|
||||
if key_id is not None and is_valid_type_id(correlation_type, type_id) and r_serv_metadata.exists('{}_metadata_{}:{}'.format(correlation_type, type_id, key_id)):
|
||||
|
||||
if date_from is None or date_to is None:
|
||||
nb_days_seen_in_pastes = 30
|
||||
else:
|
||||
# # TODO: # FIXME:
|
||||
nb_days_seen_in_pastes = 30
|
||||
|
||||
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
|
||||
|
||||
json_seen_in_paste = []
|
||||
for date in date_range_seen_in_pastes:
|
||||
nb_seen_this_day = r_serv_metadata.hget('{}:{}:{}'.format(correlation_type, type_id, date), key_id)
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)})
|
||||
|
||||
return jsonify(json_seen_in_paste)
|
||||
else:
|
||||
return jsonify()
|
||||
|
||||
@hashDecoded.route('/correlation/cryptocurrency_graph_line_json')
|
||||
def cryptocurrency_graph_line_json():
|
||||
type_id = request.args.get('type_id')
|
||||
key_id = request.args.get('key_id')
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
return correlation_graph_line_json('cryptocurrency', type_id, key_id, date_from, date_to)
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(hashDecoded, url_prefix=baseUrl)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,660 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Decoded - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
cursor: pointer;
|
||||
}
|
||||
.bar_stack:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.pie_path:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.svgText {
|
||||
pointer-events: none;
|
||||
}
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'decoded/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-10">
|
||||
<div class="mt-1" id="barchart_type">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-xl-2">
|
||||
|
||||
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
|
||||
<div class="card-body text-center py-2">
|
||||
<h6 class="card-title" style="color:#286090;">Select a date range :</h5>
|
||||
<form action="{{ url_for(correlation_type_search_endpoint) }}" id="hash_selector_form" method='post'>
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
|
||||
</div>
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
|
||||
</div>
|
||||
<div class="mt-1" style="font-size: 14px;color:#286090;">Type ID :</div>
|
||||
<select class="custom-select" name="type">
|
||||
<option>All types</option>
|
||||
{% for typ in l_type %}
|
||||
{% if type_id|string() == typ|string() %}
|
||||
<option selected>{{ typ }}</option>
|
||||
{% else %}
|
||||
<option>{{ typ }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
<div class="form-check my-1">
|
||||
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}>
|
||||
<label class="form-check-label" for="checkbox-input-show">
|
||||
<div style="color:#286090; font-size: 14px;">
|
||||
Show {{correlation_type}} <i class="fas fa-key"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fas fa-copy"></i> Search
|
||||
</button>
|
||||
<form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="pie_chart_encoded">
|
||||
</div>
|
||||
<div id="pie_chart_top5_types">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if all_metadata|length != 0 %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }} {{correlation_type}}: </h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }} {{correlation_type}}: </h3>
|
||||
{% endif %}
|
||||
<table id="tableb64" class="table table-striped table-bordered">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>type id</th>
|
||||
<th>key id</th>
|
||||
<th>first seen</th>
|
||||
<th>last seen</th>
|
||||
<th>nb item</th>
|
||||
<th>Sparkline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for key_id in all_metadata %}
|
||||
<tr>
|
||||
<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i> {{ all_metadata[key_id]['type_id'] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type_id={{ all_metadata[key_id]['type_id'] }}&key_id={{ key_id }}">{{ key_id }}</a></td>
|
||||
<td>{{ all_metadata[key_id]['first_seen'] }}</td>
|
||||
<td>{{ all_metadata[key_id]['last_seen'] }}</td>
|
||||
<td>{{ all_metadata[key_id]['nb_seen'] }}</td>
|
||||
<td id="sparklines_{{ all_metadata[key_id]['sparklines_id'] }}" style="text-align:center;"></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
{% if show_decoded_files %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }}, No Dumped Keys</h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }}, No {{correlation_type}}</h3>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_dashboard_{{correlation_type}}").addClass("active");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
},
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
},
|
||||
});
|
||||
|
||||
$('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
|
||||
$('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
|
||||
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
|
||||
{% if type_id %}
|
||||
//chart.stackBarChart =barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{daily_date}}&type_id={{type_id}}", 'id');
|
||||
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}&type_id={{type_id}}", 'id');
|
||||
{% elif type_id or daily_type_chart %}
|
||||
chart.stackBarChart =barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
|
||||
{% else %}
|
||||
chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
|
||||
{% endif %}
|
||||
|
||||
//draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
|
||||
//draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
|
||||
|
||||
chart.onResize();
|
||||
$(window).on("resize", function() {
|
||||
chart.onResize();
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
|
||||
// a sparklines plot
|
||||
function sparklines(id, points) {
|
||||
var width = 100, height = 60;
|
||||
|
||||
var data = []
|
||||
for (i = 0; i < points.length; i++) {
|
||||
data[i] = {
|
||||
'x': i,
|
||||
'y': +points[i]
|
||||
}
|
||||
}
|
||||
|
||||
var x = d3.scaleLinear()
|
||||
.range([0, width - 10])
|
||||
.domain([0,5]);
|
||||
|
||||
var y = d3.scaleLinear()
|
||||
.range([height, 0])
|
||||
.domain([0,10]);
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {return x(d.x)})
|
||||
.y(function(d) {return y(d.y)});
|
||||
|
||||
d3.select("#"+id).append('svg')
|
||||
.attr('width', width)
|
||||
.attr('height', height)
|
||||
.append('path')
|
||||
.attr('class','line')
|
||||
.datum(data)
|
||||
.attr('d', line);
|
||||
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
{% for key_id in all_metadata %}
|
||||
sparklines("sparklines_{{ all_metadata[key_id]['sparklines_id'] }}", {{ all_metadata[key_id]['sparklines_data'] }})
|
||||
{% endfor %}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var margin = {top: 20, right: 100, bottom: 55, left: 45},
|
||||
width = 1000 - margin.left - margin.right,
|
||||
height = 500 - margin.top - margin.bottom;
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x);
|
||||
|
||||
var yAxis = d3.axisLeft(y);
|
||||
|
||||
var color = d3.scaleOrdinal(d3.schemeSet3);
|
||||
|
||||
var svg = d3.select("#barchart_type").append("svg")
|
||||
.attr("id", "thesvg")
|
||||
.attr("viewBox", "0 0 1000 500")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g")
|
||||
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
function barchart_type_stack(url, id) {
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
var labelVar = 'date'; //A
|
||||
var varNames = d3.keys(data[0])
|
||||
.filter(function (key) { return key !== labelVar;}); //B
|
||||
|
||||
data.forEach(function (d) { //D
|
||||
var y0 = 0;
|
||||
d.mapping = varNames.map(function (name) {
|
||||
return {
|
||||
name: name,
|
||||
label: d[labelVar],
|
||||
y0: y0,
|
||||
y1: y0 += +d[name]
|
||||
};
|
||||
});
|
||||
d.total = d.mapping[d.mapping.length - 1].y1;
|
||||
});
|
||||
|
||||
x.domain(data.map(function (d) { return (d.date); })); //E
|
||||
y.domain([0, d3.max(data, function (d) { return d.total; })]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.attr("class", "bar")
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from="+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% else %}
|
||||
.on("click", function (d) { window.location.href = "{{ url_for(correlation_type_endpoint) }}?date_from="+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-40)" )
|
||||
{% endif %}
|
||||
.style("text-anchor", "end");
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end");
|
||||
|
||||
var selection = svg.selectAll(".series")
|
||||
.data(data)
|
||||
.enter().append("g")
|
||||
.attr("class", "series")
|
||||
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
|
||||
|
||||
selection.selectAll("rect")
|
||||
.data(function (d) { return d.mapping; })
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar_stack")
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function (d) { return y(d.y1); })
|
||||
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
|
||||
.style("fill", function (d) { return color(d.name); })
|
||||
.style("stroke", "grey")
|
||||
.on("mouseover", function (d) { showPopover.call(this, d); })
|
||||
.on("mouseout", function (d) { removePopovers(); })
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
|
||||
{% else %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for(correlation_type_endpoint) }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
|
||||
{% endif %}
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.total != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.total))
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
|
||||
{% else %}
|
||||
.on("click", function () {window.location.href = "{{ url_for(correlation_type_endpoint) }}"+'?date_from='+d.date+'&date_to='+d.date })
|
||||
{% endif %}
|
||||
.style("text-anchor", "middle")
|
||||
.text(d.total);
|
||||
}
|
||||
});
|
||||
|
||||
drawLegend(varNames);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function drawLegend (varNames) {
|
||||
var legend = svg.selectAll(".legend")
|
||||
.data(varNames.slice().reverse())
|
||||
.enter().append("g")
|
||||
.attr("class", "legend")
|
||||
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
|
||||
|
||||
legend.append("rect")
|
||||
.attr("x", 943)
|
||||
.attr("width", 10)
|
||||
.attr("height", 10)
|
||||
.style("fill", color)
|
||||
.style("stroke", "grey");
|
||||
|
||||
legend.append("text")
|
||||
.attr("class", "svgText")
|
||||
.attr("x", 941)
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".35em")
|
||||
.style("text-anchor", "end")
|
||||
.text(function (d) { return d; });
|
||||
}
|
||||
|
||||
function removePopovers () {
|
||||
$('.popover').each(function() {
|
||||
$(this).remove();
|
||||
});
|
||||
}
|
||||
|
||||
function showPopover (d) {
|
||||
$(this).popover({
|
||||
title: "<b><span id='tooltip-id-name-bar'></span></b>",
|
||||
placement: 'top',
|
||||
container: 'body',
|
||||
trigger: 'manual',
|
||||
html : true,
|
||||
content: function() {
|
||||
return "<span id='tooltip-id-label'></span>" +
|
||||
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
|
||||
});
|
||||
$(this).popover('show');
|
||||
$("#tooltip-id-name-bar").text(d.name);
|
||||
$("#tooltip-id-label").text(d.label);
|
||||
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
|
||||
}
|
||||
|
||||
chart.onResize = function () {
|
||||
var aspect = 1000 / 500, chart = $("#thesvg");
|
||||
var targetWidth = chart.parent().width();
|
||||
chart.attr("width", targetWidth);
|
||||
chart.attr("height", targetWidth / aspect);
|
||||
}
|
||||
|
||||
window.chart = chart;
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function draw_pie_chart(id, url_json, pie_on_click_url) {
|
||||
|
||||
var width_pie = 200;
|
||||
var height_pie = 200;
|
||||
var padding_pie = 10;
|
||||
var opacity_pie = .8;
|
||||
|
||||
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
|
||||
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
|
||||
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
|
||||
|
||||
var div_pie = d3.select("body").append("div")
|
||||
.attr("class", "tooltip")
|
||||
.style("opacity", 0);
|
||||
|
||||
var svg_pie = d3.select("#"+id)
|
||||
.append('svg')
|
||||
.attr("width", '100%')
|
||||
.attr("height", '100%')
|
||||
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
|
||||
.attr('preserveAspectRatio','xMinYMin')
|
||||
|
||||
|
||||
var g_pie = svg_pie.append('g')
|
||||
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
|
||||
|
||||
var arc_pie = d3.arc()
|
||||
.innerRadius(0)
|
||||
.outerRadius(radius_pie);
|
||||
|
||||
d3.json(url_json)
|
||||
.then(function(data){
|
||||
|
||||
var pie_pie = d3.pie()
|
||||
.value(function(d) { return d.value; })
|
||||
.sort(null);
|
||||
|
||||
var path_pie = g_pie.selectAll('path')
|
||||
.data(pie_pie(data))
|
||||
.enter()
|
||||
.append("g")
|
||||
.append('path')
|
||||
.attr('d', arc_pie)
|
||||
.attr('fill', (d,i) => color_pie(i))
|
||||
.attr('class', 'pie_path')
|
||||
.on("mouseover", mouseovered_pie)
|
||||
.on("mouseout", mouseouted_pie)
|
||||
.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
|
||||
.style('opacity', opacity_pie)
|
||||
.style('stroke', 'white');
|
||||
});
|
||||
|
||||
|
||||
function mouseovered_pie(d) {
|
||||
//remove old content
|
||||
$("#tooltip-id-name").remove();
|
||||
$("#tooltip-id-value").remove();
|
||||
|
||||
// tooltip
|
||||
var content;
|
||||
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
|
||||
"<br/>"+
|
||||
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
|
||||
|
||||
div_pie.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div_pie.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.data.name);
|
||||
$("#tooltip-id-value").text(d.data.value);
|
||||
}
|
||||
|
||||
function mouseouted_pie() {
|
||||
div_pie.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
<script>
|
||||
function barchart_type(url, id) {
|
||||
|
||||
|
||||
var margin = {top: 20, right: 20, bottom: 70, left: 40};
|
||||
|
||||
var width = 960 - margin.left - margin.right;
|
||||
var height = 500 - margin.top - margin.bottom;
|
||||
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x)
|
||||
//.tickFormat(d3.time.format("%Y-%m"));
|
||||
|
||||
var yAxis = d3.axisLeft(y)
|
||||
.ticks(10);
|
||||
|
||||
/*var svg = d3.select(id).append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.attr("id", "thesvg")
|
||||
.append("g")
|
||||
.attr("transform",
|
||||
"translate(" + margin.left + "," + margin.top + ")");*/
|
||||
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
x.domain(data.map(function(d) { return d.date; }));
|
||||
y.domain([0, d3.max(data, function(d) { return d.value; })]);
|
||||
|
||||
var label = svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", "-.55em")
|
||||
{% if daily_type_chart %}
|
||||
.attr("transform", "rotate(-20)" );
|
||||
{% else %}
|
||||
.attr("transform", "rotate(-70)" )
|
||||
.attr("class", "bar")
|
||||
.on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d });
|
||||
{% endif %}
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end")
|
||||
.text("Value ($)");
|
||||
|
||||
var bar = svg.selectAll("bar")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar")
|
||||
//.style("fill", "steelblue")
|
||||
.attr("x", function(d) { return x(d.date); })
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function(d) { return y(d.value); })
|
||||
.attr("height", function(d) { return height - y(d.value); })
|
||||
{% if type %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
|
||||
{% endif %}
|
||||
{% if daily_type_chart %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
|
||||
{% endif %}
|
||||
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.value != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
//.text(function(d) { return d.value; });
|
||||
.text(d.value)
|
||||
.style("text-anchor", "middle")
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.value));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -1,282 +1,287 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<head>
|
||||
<title>Decoded - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<title>HashesDecoded - AIL</title>
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<style>
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
cursor: pointer;
|
||||
}
|
||||
.bar_stack:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.pie_path:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.svgText {
|
||||
pointer-events: none;
|
||||
}
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
<style>
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
cursor: pointer;
|
||||
}
|
||||
.bar_stack:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.pie_path:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.svgText {
|
||||
pointer-events: none;
|
||||
}
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'decoded/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Hashed Files</h1>
|
||||
<div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
<div id="barchart_type">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-2">
|
||||
<div class="panel panel-info" style="text-align:center;">
|
||||
<div class="panel-heading">
|
||||
Select a date range :
|
||||
<form action="{{ url_for('hashDecoded.all_hash_search') }}" id="hash_selector_form" method='post'>
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
|
||||
</div>
|
||||
Encoding :
|
||||
<select class="form-control" name="encoding" style="width=100%;">
|
||||
<option>All encoding</option>
|
||||
{% for encod in all_encoding %}
|
||||
{% if encoding|string() == encod|string() %}
|
||||
<option selected>{{ encod }}</option>
|
||||
{% else %}
|
||||
<option>{{ encod }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
File Type :
|
||||
<select class="form-control" name="type" style="width=100%;">
|
||||
<option>All types</option>
|
||||
{% for typ in l_type %}
|
||||
{% if type|string() == typ|string() %}
|
||||
<option selected>{{ typ }}</option>
|
||||
{% else %}
|
||||
<option>{{ typ }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}>
|
||||
<div style="color:#286090; display:inline-block">
|
||||
Show decoded files <i class="fa fa-file"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fa fa-files-o"></i> Search
|
||||
</button>
|
||||
<form>
|
||||
</div>
|
||||
</div>
|
||||
<div id="pie_chart_encoded">
|
||||
</div>
|
||||
<div id="pie_chart_top5_types">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-10">
|
||||
<div class="mt-1" id="barchart_type">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- /#page-wrapper -->
|
||||
{% if l_64|length != 0 %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }} Hashed files: </h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }} Hashed files: </h3>
|
||||
{% endif %}
|
||||
<table id="tableb64" class="red_table table table-striped table-bordered">
|
||||
<thead>
|
||||
<div class="col-xl-2">
|
||||
|
||||
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
|
||||
<div class="card-body text-center py-2">
|
||||
<h6 class="card-title" style="color:#286090;">Select a date range :</h5>
|
||||
<form action="{{ url_for('hashDecoded.all_hash_search') }}" id="hash_selector_form" method='post'>
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
|
||||
</div>
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
|
||||
</div>
|
||||
<div class="mt-1" style="font-size: 14px;color:#286090;">Encoding :</div>
|
||||
<select class="custom-select" name="encoding">
|
||||
<option>All encoding</option>
|
||||
{% for encod in all_encoding %}
|
||||
{% if encoding|string() == encod|string() %}
|
||||
<option selected>{{ encod }}</option>
|
||||
{% else %}
|
||||
<option>{{ encod }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
<div class="mt-1" style="font-size: 14px;color:#286090;">File Type :</div>
|
||||
<select class="custom-select" name="type">
|
||||
<option>All types</option>
|
||||
{% for typ in l_type %}
|
||||
{% if type|string() == typ|string() %}
|
||||
<option selected>{{ typ }}</option>
|
||||
{% else %}
|
||||
<option>{{ typ }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
<div class="form-check my-1">
|
||||
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}>
|
||||
<label class="form-check-label" for="checkbox-input-show">
|
||||
<div style="color:#286090; font-size: 14px;">
|
||||
Show decoded files <i class="fas fa-file"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fas fa-copy"></i> Search
|
||||
</button>
|
||||
<form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="pie_chart_encoded">
|
||||
</div>
|
||||
<div id="pie_chart_top5_types">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if l_64|length != 0 %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }} Decoded files: </h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }} Decoded files: </h3>
|
||||
{% endif %}
|
||||
<table id="tableb64" class="table table-striped table-bordered">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>estimated type</th>
|
||||
<th>hash</th>
|
||||
<th>first seen</th>
|
||||
<th>last seen</th>
|
||||
<th>nb paste</th>
|
||||
<th>nb item</th>
|
||||
<th>size</th>
|
||||
<th>Virus Total</th>
|
||||
<th>Sparkline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fa {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ b64[2] }}">{{ b64[2] }}</a></td>
|
||||
<td>{{ b64[5] }}</td>
|
||||
<td>{{ b64[6] }}</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
<td>{{ b64[4] }}</td>
|
||||
<td style="text-align:center;max-width:150px;">
|
||||
{% if vt_enabled %}
|
||||
{% if not b64[7] %}
|
||||
<darkbutton_{{ b64[2] }}>
|
||||
<button id="submit_vt_{{ b64[2] }}" class="btn btn-primary" onclick="sendFileToVT('{{ b64[2] }}')">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</darkbutton_{{ b64[2] }}>
|
||||
{% else %}
|
||||
<a class="btn btn-primary" target="_blank" href="{{ b64[8] }}"><i class="fa fa-link"> VT Report</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
|
||||
<div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span> {{ b64[9] }}</div>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fas {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ b64[2] }}">{{ b64[2] }}</a></td>
|
||||
<td>{{ b64[5] }}</td>
|
||||
<td>{{ b64[6] }}</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
<td>{{ b64[4] }}</td>
|
||||
<td>
|
||||
{% if vt_enabled %}
|
||||
{% if not b64[7] %}
|
||||
<darkbutton_{{ b64[2] }}>
|
||||
<button id="submit_vt_{{ b64[2] }}" class="btn btn-secondary" style="font-size: 14px;" onclick="sendFileToVT('{{ b64[2] }}')">
|
||||
<i class="fas fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
</darkbutton_{{ b64[2] }}>
|
||||
{% else %}
|
||||
<a class="btn btn-secondary" target="_blank" href="{{ b64[8] }}" style="font-size: 14px;"><i class="fas fa-link"></i> VT Report</a>
|
||||
{% endif %}
|
||||
<button class="btn btn-outline-dark" onclick="updateVTReport('{{ b64[2] }}')" style="font-size: 14px;">
|
||||
<div id="report_vt_{{ b64[2] }}"><i class="fas fa-sync-alt"></i> {{ b64[9] }}</div>
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
|
||||
</td>
|
||||
<td id="sparklines_{{ b64[2] }}" style="text-align:center;">
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
{% if show_decoded_files %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }}, No Hashes</h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }}, No Hashes</h3>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td id="sparklines_{{ b64[2] }}" style="text-align:center;"></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
{% if show_decoded_files %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }}, No Hashes</h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }}, No Hashes</h3>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% endif %}
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
activePage = "page-hashDecoded"
|
||||
$("#"+activePage).addClass("active");
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_dashboard").addClass("active");
|
||||
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
|
||||
{% if type %}
|
||||
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.hash_by_type_json') }}?type={{type}}", 'id');
|
||||
{% elif daily_type_chart %}
|
||||
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
|
||||
{% else %}
|
||||
chart.stackBarChart = barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
|
||||
{% endif %}
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
|
||||
draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
|
||||
draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
|
||||
{% if type %}
|
||||
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.hash_by_type_json') }}?type={{type}}", 'id');
|
||||
{% elif daily_type_chart %}
|
||||
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
|
||||
{% else %}
|
||||
chart.stackBarChart = barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
|
||||
{% endif %}
|
||||
|
||||
draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
|
||||
draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
|
||||
|
||||
chart.onResize();
|
||||
$(window).on("resize", function() {
|
||||
chart.onResize();
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
chart.onResize();
|
||||
$(window).on("resize", function() {
|
||||
chart.onResize();
|
||||
});
|
||||
});
|
||||
</script>
|
||||
<script>
|
||||
function updateVTReport(hash) {
|
||||
//updateReport
|
||||
$.getJSON("{{ url_for('hashDecoded.update_vt_result') }}?hash="+hash,
|
||||
function(data) {
|
||||
content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
|
||||
content = '<i class="fas fa-sync-alt"></i> ' +data['report_vt']
|
||||
$( "#report_vt_"+hash ).html(content);
|
||||
});
|
||||
}
|
||||
|
|
@ -488,16 +493,19 @@ function removePopovers () {
|
|||
|
||||
function showPopover (d) {
|
||||
$(this).popover({
|
||||
title: d.name,
|
||||
placement: 'auto top',
|
||||
title: "<b><span id='tooltip-id-name-bar'></span></b>",
|
||||
placement: 'top',
|
||||
container: 'body',
|
||||
trigger: 'manual',
|
||||
html : true,
|
||||
content: function() {
|
||||
return d.label +
|
||||
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
|
||||
return "<span id='tooltip-id-label'></span>" +
|
||||
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
|
||||
});
|
||||
$(this).popover('show')
|
||||
$(this).popover('show');
|
||||
$("#tooltip-id-name-bar").text(d.name);
|
||||
$("#tooltip-id-label").text(d.label);
|
||||
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
|
||||
}
|
||||
|
||||
chart.onResize = function () {
|
||||
|
|
@ -566,13 +574,16 @@ function draw_pie_chart(id, url_json, pie_on_click_url) {
|
|||
|
||||
|
||||
function mouseovered_pie(d) {
|
||||
//remove old content
|
||||
$("#tooltip-id-name").remove();
|
||||
$("#tooltip-id-value").remove();
|
||||
|
||||
// tooltip
|
||||
var content;
|
||||
|
||||
content = "<b>"+d.data.name+"</b>"+"<br/>"+
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
|
||||
"<br/>"+
|
||||
"<i>Decoded</i>: "+d.data.value+"<br/>"
|
||||
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
|
||||
|
||||
div_pie.transition()
|
||||
.duration(200)
|
||||
|
|
@ -580,6 +591,9 @@ function draw_pie_chart(id, url_json, pie_on_click_url) {
|
|||
div_pie.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.data.name);
|
||||
$("#tooltip-id-value").text(d.data.value);
|
||||
}
|
||||
|
||||
function mouseouted_pie() {
|
||||
|
|
@ -692,6 +706,6 @@ function barchart_type(url, id) {
|
|||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,553 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
|
||||
<title>AIL - framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
line.link {
|
||||
stroke: #666;
|
||||
}
|
||||
line.link:hover{
|
||||
stroke: red;
|
||||
stroke-width: 2px
|
||||
}
|
||||
.line_sparkline {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.node {
|
||||
pointer-events: all;
|
||||
}
|
||||
|
||||
circle {
|
||||
stroke: none;
|
||||
}
|
||||
|
||||
.graph_text_node {
|
||||
font: 8px sans-serif;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.graph_node_icon {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.node text {
|
||||
font: 8px sans-serif;
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
|
||||
.graph_panel {
|
||||
padding: unset;
|
||||
}
|
||||
|
||||
.line_graph {
|
||||
fill: none;
|
||||
stroke: steelblue;
|
||||
stroke-width: 2px;
|
||||
stroke-linejoin: round;
|
||||
stroke-linecap: round;
|
||||
stroke-width: 1.5;
|
||||
/*attr('stroke', '#bcbd22').*/
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'decoded/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h4 class="text-secondary">{{ key_id }} :</h4>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>type</th>
|
||||
<th>First_seen</th>
|
||||
<th>Last_seen</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><i class="{{ type_icon }}"></i> {{ type_id }}</td>
|
||||
<td>{{ key_id_metadata['first_seen'] }}</td>
|
||||
<td>{{ key_id_metadata['last_seen'] }}</td>
|
||||
<td>{{ key_id_metadata['nb_seen'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-md-1">
|
||||
<div id="sparkline"></div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-10">
|
||||
|
||||
<div class="card mb-3">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-project-diagram"></i> Graph
|
||||
</div>
|
||||
<div class="card-body graph_panel">
|
||||
<div id="graph">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-xl-2">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-project-diagram"></i> Graph
|
||||
</div>
|
||||
<div class="card-body text-center px-0 py-0">
|
||||
<button class="btn btn-primary my-4" onclick="resize_graph();">
|
||||
<i class="fas fa-sync"></i> Resize Graph
|
||||
</button>
|
||||
|
||||
{% if correlation_type=='pgpdump' %}
|
||||
{% include 'decoded/show_helper_pgpdump.html' %}
|
||||
{% elif correlation_type=='cryptocurrency' %}
|
||||
{% include 'decoded/show_helper_cryptocurrency.html' %}
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-chart-bar"></i> Graph
|
||||
</div>
|
||||
<div class="panel-body ">
|
||||
<div id="graph_line">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var all_graph = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
sparklines("sparkline", {{ sparkline_values }})
|
||||
|
||||
all_graph.node_graph = create_graph("{{ url_for(graph_node_endpoint) }}?type_id={{type_id}}&key_id={{key_id}}");
|
||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for(graph_line_endpoint) }}?type_id={{type_id}}&key_id={{key_id}}");
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function resize_graph() {
|
||||
zoom.translateTo(svg_node, 200, 200);
|
||||
zoom.scaleTo(svg_node, 2);
|
||||
}
|
||||
|
||||
</script>
|
||||
<script>
|
||||
//var data = [6,3,3,2,5,3,9];
|
||||
|
||||
// a sparklines plot
|
||||
function sparklines(id, points) {
|
||||
var width_spark = 100, height_spark = 60;
|
||||
|
||||
var data = []
|
||||
for (i = 0; i < points.length; i++) {
|
||||
data[i] = {
|
||||
'x': i,
|
||||
'y': +points[i]
|
||||
}
|
||||
}
|
||||
|
||||
var x = d3.scaleLinear()
|
||||
.range([0, width_spark - 10])
|
||||
.domain([0,5]);
|
||||
|
||||
var y = d3.scaleLinear()
|
||||
.range([height_spark, 0])
|
||||
.domain([0,10]);
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {return x(d.x)})
|
||||
.y(function(d) {return y(d.y)});
|
||||
|
||||
d3.select("#"+id).append('svg')
|
||||
.attr('width', width_spark)
|
||||
.attr('height', height_spark)
|
||||
.append('path')
|
||||
.attr('class','line_sparkline')
|
||||
.datum(data)
|
||||
.attr('d', line);
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var width = 400,
|
||||
height = 400;
|
||||
|
||||
var link;
|
||||
|
||||
var zoom = d3.zoom()
|
||||
.scaleExtent([.2, 10])
|
||||
.on("zoom", zoomed);
|
||||
|
||||
//var transform = d3.zoomIdentity;
|
||||
|
||||
var color = d3.scaleOrdinal(d3.schemeCategory10);
|
||||
|
||||
var div = d3.select("body").append("div")
|
||||
.attr("class", "tooltip")
|
||||
.style("opacity", 0);
|
||||
|
||||
var simulation = d3.forceSimulation()
|
||||
.force("link", d3.forceLink().id(function(d) { return d.id; }))
|
||||
.force("charge", d3.forceManyBody())
|
||||
.force("center", d3.forceCenter(width / 2, height / 2));
|
||||
//.on("tick", ticked);
|
||||
|
||||
var svg_node = d3.select("#graph").append("svg")
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width)
|
||||
.attr("height", height)
|
||||
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
|
||||
.on("dblclick.zoom", null)
|
||||
|
||||
var container_graph = svg_node.append("g");
|
||||
//.attr("transform", "translate(40,0)")
|
||||
//.attr("transform", "scale(2)");
|
||||
|
||||
function create_graph(url){
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
link = container_graph.append("g")
|
||||
.selectAll("line")
|
||||
.data(data.links)
|
||||
.enter().append("line")
|
||||
.attr("class", "link");
|
||||
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
|
||||
|
||||
var node = container_graph.selectAll(".node")
|
||||
.data(data.nodes)
|
||||
.enter().append("g")
|
||||
.attr("class", "nodes")
|
||||
.on("dblclick", doubleclick)
|
||||
.on("click", click)
|
||||
.on("mouseover", mouseovered)
|
||||
.on("mouseout", mouseouted)
|
||||
.call(d3.drag()
|
||||
.on("start", drag_start)
|
||||
.on("drag", dragged)
|
||||
.on("end", drag_end));
|
||||
|
||||
|
||||
node.append("circle")
|
||||
.attr("r", function(d) {
|
||||
return (d.hash) ? 6 : 5; })
|
||||
.attr("fill", function(d) {
|
||||
if(!d.hash){ return color(d.group);}
|
||||
if(d.group == 1){ return "orange";}
|
||||
return "rgb(141, 211, 199)"; });
|
||||
|
||||
node.append('text')
|
||||
.attr('text-anchor', 'middle')
|
||||
.attr('dominant-baseline', 'central')
|
||||
.attr("class", "graph_node_icon {{font_family}}")
|
||||
.attr('font-size', '8px' )
|
||||
.attr('pointer-events', 'none')
|
||||
.text(function(d) {
|
||||
if(d.hash){
|
||||
return d.icon
|
||||
} });
|
||||
|
||||
zoom.translateTo(svg_node, 200, 200);
|
||||
zoom.scaleTo(svg_node, 2);
|
||||
|
||||
/* node.append("text")
|
||||
.attr("dy", 3)
|
||||
.attr("dx", 7)
|
||||
.attr("class", "graph_text_node")
|
||||
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
|
||||
.text(function(d) { return d.id; });*/
|
||||
|
||||
simulation
|
||||
.nodes(data.nodes)
|
||||
.on("tick", ticked);
|
||||
|
||||
simulation.force("link")
|
||||
.links(data.links);
|
||||
|
||||
function ticked() {
|
||||
link
|
||||
.attr("x1", function(d) { return d.source.x; })
|
||||
.attr("y1", function(d) { return d.source.y; })
|
||||
.attr("x2", function(d) { return d.target.x; })
|
||||
.attr("y2", function(d) { return d.target.y; });
|
||||
|
||||
/*node
|
||||
.attr("cx", function(d) { return d.x; })
|
||||
.attr("cy", function(d) { return d.y; });*/
|
||||
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
function zoomed() {
|
||||
container_graph.attr("transform", d3.event.transform);
|
||||
}
|
||||
|
||||
function doubleclick (d) {
|
||||
window.open(d.url, '_blank');
|
||||
}
|
||||
|
||||
function click (d) {
|
||||
console.log('clicked')
|
||||
}
|
||||
|
||||
function drag_start(d) {
|
||||
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
|
||||
d.fx = d.x;
|
||||
d.fy = d.y;
|
||||
}
|
||||
|
||||
function dragged(d) {
|
||||
d.fx = d3.event.x;
|
||||
d.fy = d3.event.y;
|
||||
}
|
||||
|
||||
function drag_end(d) {
|
||||
if (!d3.event.active) simulation.alphaTarget(0);
|
||||
d.fx = d.x;
|
||||
d.fy = d.y;
|
||||
}
|
||||
|
||||
function mouseovered(d) {
|
||||
|
||||
// tooltip
|
||||
var content;
|
||||
|
||||
if(d.hash == true){
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
|
||||
"<br/>"+
|
||||
"<i>First seen</i>: <span id='tooltip-id-first_seen'></span><br/>"+
|
||||
"<i>Last seen</i>: <span id='tooltip-id-last_seen'></span><br/>"+
|
||||
"<i>nb_seen</i>: <span id='tooltip-id-nb_seen'></span>";
|
||||
|
||||
div.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.id);
|
||||
$("#tooltip-id-first_seen").text(d.first_seen);
|
||||
$("#tooltip-id-last_seen").text(d.last_seen);
|
||||
$("#tooltip-id-nb_seen").text(d.nb_seen_in_paste);
|
||||
|
||||
} else {
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>";
|
||||
|
||||
div.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.id);
|
||||
|
||||
}
|
||||
|
||||
//links
|
||||
/*link.style("stroke-opacity", function(o) {
|
||||
return o.source === d || o.target === d ? 1 : opacity;
|
||||
});*/
|
||||
link.style("stroke", function(o){
|
||||
return o.source === d || o.target === d ? "#666" : "#ddd";
|
||||
});
|
||||
}
|
||||
|
||||
function mouseouted() {
|
||||
div.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
|
||||
link.style("stroke", "#666");
|
||||
}
|
||||
|
||||
all_graph.onResize = function () {
|
||||
var aspect = 1000 / 500, all_graph = $("#graph_div");
|
||||
var targetWidth = all_graph.parent().width();
|
||||
all_graph.attr("width", targetWidth);
|
||||
all_graph.attr("height", targetWidth / aspect);
|
||||
}
|
||||
|
||||
window.all_graph = all_graph;
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function create_line_chart(id, url){
|
||||
|
||||
var width = 900;
|
||||
var height = Math.round(width / 4);
|
||||
|
||||
var margin = {top: 20, right: 55, bottom: 50, left: 40};
|
||||
|
||||
var x = d3.scaleTime().range([0, width]);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x);
|
||||
var yAxis = d3.axisLeft(y);
|
||||
|
||||
var parseTime = d3.timeParse("%Y-%m-%d");
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {
|
||||
return x(d.date);
|
||||
}).y(function(d) {
|
||||
return y(d.value);
|
||||
});
|
||||
|
||||
var svg_line = d3.select('#'+id).append('svg')
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append('g')
|
||||
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
|
||||
|
||||
var div = d3.select('body').append('div')
|
||||
.attr('class', 'tooltip')
|
||||
.style('opacity', 0);
|
||||
|
||||
//add div tooltip
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.date_label = d.date;
|
||||
d.date = parseTime(d.date);
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
// fit the data
|
||||
x.domain(d3.extent(data, function(d) { return d.date; }));
|
||||
//x.domain(data.map(function (d) { return d.date; })); //E
|
||||
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
|
||||
|
||||
//line
|
||||
svg_line.append("path")
|
||||
.data([data])
|
||||
.attr("class", "line_graph")
|
||||
.attr("d", line);
|
||||
|
||||
// add X axis
|
||||
svg_line.append("g")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(d3.axisBottom(x))
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("transform", "rotate(-45)" );
|
||||
|
||||
// Add the Y Axis
|
||||
svg_line.append("g")
|
||||
.call(d3.axisLeft(y));
|
||||
|
||||
//add a dot circle
|
||||
svg_line.selectAll('dot')
|
||||
.data(data).enter()
|
||||
.append('circle')
|
||||
.attr('r', 2)
|
||||
.attr('cx', function(d) { return x(d.date); })
|
||||
.attr('cy', function(d) { return y(d.value); })
|
||||
|
||||
.on('mouseover', function(d) {
|
||||
div.transition().style('opacity', .9);
|
||||
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
})
|
||||
.on('mouseout', function(d)
|
||||
{
|
||||
div.transition().style('opacity', 0);
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -3,27 +3,20 @@
|
|||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Hash Information - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
line.link {
|
||||
stroke: #666;
|
||||
|
|
@ -88,178 +81,191 @@
|
|||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'decoded/menu_sidebar.html' %}
|
||||
|
||||
</div>
|
||||
|
||||
<!-- /#page-wrapper -->
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading panelText">
|
||||
<h3>{{ hash }} :</h3>
|
||||
<span class="pull-right"> </span>
|
||||
<span class="badge pull-right">6 / 26</span>
|
||||
<ul class="list-group"><li class="list-group-item">
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h4 class="text-secondary">{{ hash }} :</h4>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
|
||||
<table class="table table-condensed">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Estimated type</th>
|
||||
<th>First_seen</th>
|
||||
<th>Last_seen</th>
|
||||
<th>Size (Kb)</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="panelText"><i class="fa {{ file_icon }}"></i> {{ estimated_type }}</td>
|
||||
<td class="panelText">{{ first_seen }}</td>
|
||||
<td class="panelText">{{ last_seen }}</td>
|
||||
<td class="panelText">{{ size }}</td>
|
||||
<td class="panelText">{{ nb_seen_in_all_pastes }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Estimated type</th>
|
||||
<th>First_seen</th>
|
||||
<th>Last_seen</th>
|
||||
<th>Size (Kb)</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><i class="fas {{ file_icon }}"></i> {{ estimated_type }}</td>
|
||||
<td>{{ first_seen }}</td>
|
||||
<td>{{ last_seen }}</td>
|
||||
<td>{{ size }}</td>
|
||||
<td>{{ nb_seen_in_all_pastes }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-md-1">
|
||||
<div id="sparkline"></div>
|
||||
</div>
|
||||
</div>
|
||||
</li></ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
{% if vt_enabled %}
|
||||
{% if not b64_vt %}
|
||||
<darkbutton>
|
||||
<button id="submit_vt_b" class="btn btn-primary" onclick="sendFileToVT('{{ hash }}')">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</darkbutton>
|
||||
{% else %}
|
||||
<a class="btn btn-primary" target="_blank" href="{{ b64_vt_link }}"><i class="fa fa-link"> VT Report</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ hash }}')">
|
||||
<div id="report_vt_b"><span class="glyphicon glyphicon-refresh"></span> {{ b64_vt_report }}</div>
|
||||
</button>
|
||||
{% if vt_enabled %}
|
||||
{% if not b64_vt %}
|
||||
<darkbutton>
|
||||
<button id="submit_vt_b" class="btn btn-primary" onclick="sendFileToVT('{{ hash }}')" style="font-size: 15px">
|
||||
<i class="fas fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</darkbutton>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
<a class="btn btn-primary" target="_blank" href="{{ b64_vt_link }}" style="font-size: 15px"><i class="fas fa-link"></i> VT Report</a>
|
||||
{% endif %}
|
||||
<button class="btn btn-outline-secondary" onclick="updateVTReport('{{ hash }}')" style="font-size: 15px">
|
||||
<div id="report_vt_b"><i class="fas fa-sync-alt"></i> {{ b64_vt_report }}</div>
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
|
||||
<a href="{{ url_for('hashDecoded.downloadHash') }}?hash={{hash}}" target="blank">
|
||||
<button class='btn btn-info pull-right'><i id="flash-tld" class="glyphicon glyphicon-download-alt " flash-tld=""></i> Download Hashed file
|
||||
</button>
|
||||
</a>
|
||||
</div></div>
|
||||
<a href="{{ url_for('hashDecoded.downloadHash') }}?hash={{hash}}" target="blank" class="float-right" style="font-size: 15px">
|
||||
<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
|
||||
</button>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
|
||||
</div>
|
||||
<div class="panel-body graph_panel">
|
||||
<div id="graph">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-2">
|
||||
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-unlock-alt" aria-hidden="true"></i> Encoding
|
||||
</div>
|
||||
<div class="panel-body" style="text-align:center;">
|
||||
{% for encoding in list_hash_decoder %}
|
||||
<button id="" class="btn btn-default">
|
||||
{{encoding['encoding']}} <span class="badge">{{encoding['nb_seen']}}</span>
|
||||
</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
|
||||
</div>
|
||||
<div class="panel-body" style="text-align:center;">
|
||||
<button class="btn btn-primary" onclick="resize_graph();">
|
||||
<span class="glyphicon glyphicon-refresh"></span> Resize Graph</div>
|
||||
</button>
|
||||
|
||||
|
||||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info" style="text-align:center;"><i class="fa fa-info-circle fa-2x"></i></li>
|
||||
<li class="list-group-item">
|
||||
<p>Double click on a node to open Hash/Paste<br><br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
|
||||
Current Hash<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
|
||||
Hashes<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
|
||||
Pastes
|
||||
</p>
|
||||
</li>
|
||||
<li class="list-group-item list-group-item-info">
|
||||
Hash Types:
|
||||
</li>
|
||||
<li class="list-group-item">
|
||||
<i class="fa fa-file"></i> Application<br>
|
||||
<i class="fa fa-file-video-o"></i> Audio<br>
|
||||
<i class="fa fa-file-image-o"></i> Image<br>
|
||||
<i class="fa fa-file-text-o"></i> Text<br>
|
||||
<i class="fa fa-file-o"></i> Other
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-stats" flash-tld=""></i> Graph
|
||||
</div>
|
||||
<div class="panel-body ">
|
||||
<div id="graph_line">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-xl-10">
|
||||
|
||||
<div class="card mb-3">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-project-diagram"></i> Graph
|
||||
</div>
|
||||
<div class="card-body graph_panel">
|
||||
<div id="graph">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-xl-2">
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-unlock-alt" aria-hidden="true"></i> Encoding
|
||||
</div>
|
||||
<div class="card-body text-center">
|
||||
{% for encoding in list_hash_decoder %}
|
||||
<button class="btn btn-outline-dark" disabled>
|
||||
{{encoding['encoding']}} <span class="badge badge-dark">{{encoding['nb_seen']}}</span>
|
||||
</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-project-diagram"></i> Graph
|
||||
</div>
|
||||
<div class="card-body text-center px-0 py-0">
|
||||
<button class="btn btn-primary my-4" onclick="resize_graph();">
|
||||
<i class="fas fa-sync"></i> Resize Graph
|
||||
</button>
|
||||
|
||||
|
||||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
|
||||
<li class="list-group-item text-left">
|
||||
<p>Double click on a node to open Hash/Paste<br><br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
|
||||
Current Hash<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
|
||||
Hashes<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
|
||||
Pastes
|
||||
</p>
|
||||
</li>
|
||||
<li class="list-group-item list-group-item-info">
|
||||
Hash Types:
|
||||
</li>
|
||||
<li class="list-group-item text-left">
|
||||
<i class="fas fa-file"></i> Application<br>
|
||||
<i class="fas fa-file-video"></i> Audio<br>
|
||||
<i class="fas fa-file-image"></i> Image<br>
|
||||
<i class="fas fa-file-alt"></i> Text<br>
|
||||
<i class="fas fa-sticky-note"></i> Other
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<i class="fas fa-chart-bar"></i> Graph
|
||||
</div>
|
||||
<div class="panel-body ">
|
||||
<div id="graph_line">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var all_graph = {};
|
||||
$(document).ready(function(){
|
||||
sparklines("sparkline", {{ sparkline_values }})
|
||||
<script>
|
||||
var all_graph = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
sparklines("sparkline", {{ sparkline_values }})
|
||||
|
||||
all_graph.node_graph = create_graph("{{ url_for('hashDecoded.hash_graph_node_json') }}?hash={{hash}}");
|
||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{hash}}");
|
||||
all_graph.onResize();
|
||||
});
|
||||
all_graph.node_graph = create_graph("{{ url_for('hashDecoded.hash_graph_node_json') }}?hash={{hash}}");
|
||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{hash}}");
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
});
|
||||
</script>
|
||||
<script>
|
||||
function sendFileToVT(hash) {
|
||||
//send file to vt
|
||||
$.getJSON("{{ url_for('hashDecoded.send_file_to_vt_js') }}?hash="+hash,
|
||||
function(data) {
|
||||
var content = '<a id="submit_vt_b" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
|
||||
var content = '<a id="submit_vt_b" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fas fa-link"> '+ ' VT Report' +'</i></a>';
|
||||
$('#submit_vt_b').remove();
|
||||
$('darkbutton').append(content);
|
||||
});
|
||||
|
|
@ -269,7 +275,7 @@
|
|||
//updateReport
|
||||
$.getJSON("{{ url_for('hashDecoded.update_vt_result') }}?hash="+hash,
|
||||
function(data) {
|
||||
var content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt'];
|
||||
var content = '<i class="fas fa-sync-alt"></i> ' +data['report_vt'];
|
||||
$( "#report_vt_b" ).html(content);
|
||||
});
|
||||
}
|
||||
|
|
@ -392,8 +398,7 @@ d3.json(url)
|
|||
node.append('text')
|
||||
.attr('text-anchor', 'middle')
|
||||
.attr('dominant-baseline', 'central')
|
||||
.attr("class", "graph_node_icon")
|
||||
.attr('font-family', 'FontAwesome')
|
||||
.attr("class", "graph_node_icon fa")
|
||||
.attr('font-size', '8px' )
|
||||
.attr('pointer-events', 'none')
|
||||
.text(function(d) {
|
||||
|
|
@ -469,24 +474,36 @@ function mouseovered(d) {
|
|||
var content;
|
||||
|
||||
if(d.hash == true){
|
||||
content = "<b>"+d.id+"</b>"+"<br/>"+
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
|
||||
"<br/>"+
|
||||
"<i>First seen</i>: "+d.first_seen+"<br/>"+
|
||||
"<i>Last seen</i>: "+d.last_seen+"<br/>"+
|
||||
"<i>nb_seen_in_paste</i>: "+d.nb_seen_in_paste+"<br/>"+
|
||||
"<i>Size (kb)</i>: "+d.size+"<br/>"+
|
||||
"<br/>"+
|
||||
"<i>Estimated type</i>: "+d.estimated_type;
|
||||
} else {
|
||||
content = "<b>"+d.id+"</b>"+"<br/>";
|
||||
}
|
||||
"<i>First seen</i>: <span id='tooltip-id-first_seen'></span><br/>"+
|
||||
"<i>Last seen</i>: <span id='tooltip-id-last_seen'></span><br/>"+
|
||||
"<i>nb_seen</i>: <span id='tooltip-id-nb_seen'></span>";
|
||||
|
||||
div.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.id);
|
||||
$("#tooltip-id-first_seen").text(d.first_seen);
|
||||
$("#tooltip-id-last_seen").text(d.last_seen);
|
||||
$("#tooltip-id-nb_seen").text(d.nb_seen_in_paste);
|
||||
|
||||
} else {
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>";
|
||||
|
||||
div.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.id);
|
||||
}
|
||||
|
||||
//links
|
||||
/*link.style("stroke-opacity", function(o) {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ baseUrl = Flask_config.baseUrl
|
|||
r_cache = Flask_config.r_cache
|
||||
r_serv_onion = Flask_config.r_serv_onion
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
crawler_enabled = Flask_config.crawler_enabled
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
|
||||
|
|
@ -117,7 +118,12 @@ def get_type_domain(domain):
|
|||
def get_domain_from_url(url):
|
||||
faup.decode(url)
|
||||
unpack_url = faup.get()
|
||||
domain = unpack_url['domain'].decode()
|
||||
domain = unpack_url['domain']
|
||||
## TODO: FIXME remove me
|
||||
try:
|
||||
domain = domain.decode()
|
||||
except:
|
||||
pass
|
||||
return domain
|
||||
|
||||
def get_last_domains_crawled(type):
|
||||
|
|
@ -245,6 +251,7 @@ def dashboard():
|
|||
statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
|
||||
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
|
||||
crawler_enabled=crawler_enabled,
|
||||
crawler_metadata_regular=crawler_metadata_regular,
|
||||
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
|
||||
|
||||
|
|
@ -256,7 +263,7 @@ def hiddenServices_page_test():
|
|||
@hiddenServices.route("/crawlers/manual", methods=['GET'])
|
||||
@login_required
|
||||
def manual():
|
||||
return render_template("Crawler_Splash_manual.html")
|
||||
return render_template("Crawler_Splash_manual.html", crawler_enabled=crawler_enabled)
|
||||
|
||||
@hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
|
||||
@login_required
|
||||
|
|
@ -296,6 +303,7 @@ def Crawler_Splash_last_by_type():
|
|||
crawler_metadata = get_crawler_splash_status(type)
|
||||
|
||||
return render_template("Crawler_Splash_last_by_type.html", type=type, type_name=type_name,
|
||||
crawler_enabled=crawler_enabled,
|
||||
last_domains=list_domains, statDomains=statDomains,
|
||||
crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
|
||||
|
||||
|
|
@ -429,8 +437,19 @@ def create_spider_splash():
|
|||
# get service_type
|
||||
faup.decode(url)
|
||||
unpack_url = faup.get()
|
||||
domain = unpack_url['domain'].decode()
|
||||
if unpack_url['tld'] == b'onion':
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
domain = unpack_url['domain'].decode()
|
||||
except:
|
||||
domain = unpack_url['domain']
|
||||
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
tld = unpack_url['tld'].decode()
|
||||
except:
|
||||
tld = unpack_url['tld']
|
||||
|
||||
if tld == 'onion':
|
||||
service_type = 'onion'
|
||||
else:
|
||||
service_type = 'regular'
|
||||
|
|
@ -503,6 +522,7 @@ def auto_crawler():
|
|||
|
||||
return render_template("Crawler_auto.html", page=page, nb_page_max=nb_page_max,
|
||||
last_domains=last_domains,
|
||||
crawler_enabled=crawler_enabled,
|
||||
auto_crawler_domain_onions_metadata=auto_crawler_domain_onions_metadata,
|
||||
auto_crawler_domain_regular_metadata=auto_crawler_domain_regular_metadata)
|
||||
|
||||
|
|
@ -712,10 +732,19 @@ def show_domain():
|
|||
port = request.args.get('port')
|
||||
faup.decode(domain)
|
||||
unpack_url = faup.get()
|
||||
domain = unpack_url['domain'].decode()
|
||||
|
||||
## TODO: # FIXME: remove me
|
||||
try:
|
||||
domain = unpack_url['domain'].decode()
|
||||
except:
|
||||
domain = unpack_url['domain']
|
||||
|
||||
if not port:
|
||||
if unpack_url['port']:
|
||||
port = unpack_url['port'].decode()
|
||||
try:
|
||||
port = unpack_url['port'].decode()
|
||||
except:
|
||||
port = unpack_url['port']
|
||||
else:
|
||||
port = 80
|
||||
try:
|
||||
|
|
@ -815,7 +844,7 @@ def domain_crawled_by_type_json():
|
|||
day_crawled = {}
|
||||
day_crawled['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
day_crawled['UP']= nb_domain_up = r_serv_onion.scard('{}_up:{}'.format(type, date))
|
||||
day_crawled['DOWN']= nb_domain_up = r_serv_onion.scard('{}_up:{}'.format(type, date))
|
||||
day_crawled['DOWN']= nb_domain_up = r_serv_onion.scard('{}_down:{}'.format(type, date))
|
||||
range_decoder.append(day_crawled)
|
||||
|
||||
return jsonify(range_decoder)
|
||||
|
|
|
|||
|
|
@ -46,6 +46,8 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-12 col-xl-6">
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
|
||||
<div class="card text-white bg-dark mb-3 mt-1">
|
||||
<div class="card-header">
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
|
||||
{%if last_domains%}
|
||||
<div class="table-responsive mt-1 mb-3 table-hover table-borderless table-striped">
|
||||
<table class="table">
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'crawler/crawler_disabled.html' %}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-6">
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,8 @@ def get_item_screenshot_path(item):
|
|||
screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot')
|
||||
if screenshot:
|
||||
screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
|
||||
else:
|
||||
screenshot = ''
|
||||
return screenshot
|
||||
|
||||
def showpaste(content_range, requested_path):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
{% if not crawler_enabled %}
|
||||
<div class="alert alert-secondary text-center my-2" role="alert">
|
||||
<h1><i class="fas fa-times-circle text-danger"></i> Crawler Disabled</h1>
|
||||
<p>...</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
|
||||
|
||||
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
|
||||
<i class="fas fa-align-left"></i>
|
||||
<span>Toggle Sidebar</span>
|
||||
</button>
|
||||
|
||||
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
|
||||
<h5 class="d-flex text-muted w-100">
|
||||
<span>Items Decoded </span>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
|
||||
<i class="fas fa-search"></i>
|
||||
<span>Dashboard</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100">
|
||||
<span>PGP Dumps</span>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump">
|
||||
<i class="fas fa-key"></i>
|
||||
<span>PGP Dumps</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100">
|
||||
<span>Cryptocurrency</span>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency">
|
||||
<i class="fas fa-coins"></i>
|
||||
<span>Cryptocurrency</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
|
||||
<li class="list-group-item text-left">
|
||||
<p>Double click on a node to open Cryptocurrency/Paste<br><br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
|
||||
Current Cryptocurrency<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
|
||||
Cryptocurrency<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
|
||||
Pastes
|
||||
</p>
|
||||
</li>
|
||||
<li class="list-group-item list-group-item-info">
|
||||
Cryptocurrency Types:
|
||||
</li>
|
||||
<li class="list-group-item text-left">
|
||||
<i class="fab fa-btc"></i> Bitcoin<br>
|
||||
<i class="fab fa-monero"></i> Monero<br>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
|
||||
<li class="list-group-item text-left">
|
||||
<p>Double click on a node to open PgpDump/Paste<br><br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
|
||||
Current PgpDump<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
|
||||
PgpDump<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
|
||||
Pastes
|
||||
</p>
|
||||
</li>
|
||||
<li class="list-group-item list-group-item-info">
|
||||
PgpDump Types:
|
||||
</li>
|
||||
<li class="list-group-item text-left">
|
||||
<i class="fas fa-key"></i> Key<br>
|
||||
<i class="fas fa-user-tag"></i> Name<br>
|
||||
<i class="fas fa-at"></i> Mail<br>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
@ -25,7 +25,7 @@
|
|||
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-lock-open"></i> Decoded</a>
|
||||
<a class="nav-link" id="page-Decoded" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-lock-open"></i> Decoded</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>
|
||||
|
|
|
|||
Loading…
Reference in New Issue