mirror of https://github.com/CIRCL/AIL-framework
add apiKeys module
Terrtia
parent
29c368250e
commit
c3428df5dc
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python3.5
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The ApiKey Module
|
||||
======================
|
||||
|
||||
This module is consuming the Redis-list created by the Categ module.
|
||||
|
||||
It apply API_key regexes on paste content and warn if above a threshold.
|
||||
|
||||
"""
|
||||
|
||||
import redis
|
||||
import pprint
|
||||
import time
|
||||
import re
|
||||
|
||||
from packages import Paste
|
||||
from packages import lib_refine
|
||||
from pubsublogger import publisher
|
||||
|
||||
from Helper import Process
|
||||
|
||||
|
||||
def search_api_key(message):
|
||||
filename, score = message.split()
|
||||
paste = Paste.Paste(filename)
|
||||
content = paste.get_p_content()
|
||||
|
||||
aws_access_key = regex_aws_access_key.findall(content)
|
||||
aws_secret_key = regex_aws_secret_key.findall(content)
|
||||
google_api_key = regex_google_api_key.findall(content)
|
||||
|
||||
print(aws_access_key)
|
||||
print(aws_secret_key)
|
||||
print(google_api_key)
|
||||
|
||||
if(len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0):
|
||||
|
||||
print('-------------------------------')
|
||||
print(aws_access_key)
|
||||
print(aws_secret_key)
|
||||
print(google_api_key)
|
||||
|
||||
to_print = 'ApiKey;{};{};{};'.format(
|
||||
paste.p_source, paste.p_date, paste.p_name)
|
||||
if(len(google_api_key) > 0):
|
||||
print('found google api key')
|
||||
print(to_print)
|
||||
publisher.warning('{}Checked {} found Google API Key;{}'.format(
|
||||
to_print, len(google_api_key), paste.p_path))
|
||||
|
||||
if(len(aws_access_key) > 0 or len(aws_secret_key) > 0):
|
||||
print('found AWS key')
|
||||
print(to_print)
|
||||
total = len(aws_access_key) + len(aws_secret_key)
|
||||
publisher.warning('{}Checked {} found AWS Key;{}'.format(
|
||||
to_print, total, paste.p_path))
|
||||
|
||||
|
||||
msg = 'apikey;{}'.format(filename)
|
||||
p.populate_set_out(msg, 'alertHandler')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(filename, 'Duplicate')
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = 'ApiKey'
|
||||
|
||||
p = Process(config_section)
|
||||
|
||||
publisher.info("ApiKey started")
|
||||
|
||||
# REDIS #
|
||||
r_serv2 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Cache", "host"),
|
||||
port=p.config.getint("Redis_Cache", "port"),
|
||||
db=p.config.getint("Redis_Cache", "db"))
|
||||
|
||||
message = p.get_from_set()
|
||||
|
||||
# TODO improve REGEX
|
||||
regex_aws_access_key = re.compile(r'(?<![A-Z0-9])=[A-Z0-9]{20}(?![A-Z0-9])')
|
||||
regex_aws_secret_key = re.compile(r'(?<!=[A-Za-z0-9+])=[A-Za-z0-9+]{40}(?![A-Za-z0-9+])')
|
||||
|
||||
regex_google_api_key = re.compile(r'=AIza[0-9a-zA-Z-_]{35}')
|
||||
|
||||
while True:
|
||||
|
||||
message = p.get_from_set()
|
||||
|
||||
if message is not None:
|
||||
|
||||
search_api_key(message)
|
||||
|
||||
|
||||
else:
|
||||
publisher.debug("Script ApiKey is Idling 10s")
|
||||
#print('Sleeping')
|
||||
time.sleep(10)
|
||||
|
|
@ -67,7 +67,7 @@ if __name__ == "__main__":
|
|||
# FUNCTIONS #
|
||||
publisher.info("Script Categ started")
|
||||
|
||||
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'Dox']
|
||||
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'ApiKey']
|
||||
tmp_dict = {}
|
||||
for filename in categories:
|
||||
bname = os.path.basename(filename)
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ if __name__ == "__main__":
|
|||
site_occurence = re.findall(regex_site_for_stats, content)
|
||||
for site in site_occurence:
|
||||
site_domain = site[1:-1]
|
||||
if site_domain in creds_sites.keys():
|
||||
if site_domain.encode('utf8') in creds_sites.keys():
|
||||
creds_sites[site_domain] += 1
|
||||
else:
|
||||
creds_sites[site_domain] = 1
|
||||
|
|
@ -132,6 +132,11 @@ if __name__ == "__main__":
|
|||
creds_sites[domain] = 1
|
||||
|
||||
for site, num in creds_sites.items(): # Send for each different site to moduleStats
|
||||
try:
|
||||
site = site.decode('utf8')
|
||||
except:
|
||||
pass
|
||||
|
||||
mssg = 'credential;{};{};{}'.format(num, site, paste.p_date)
|
||||
print(mssg)
|
||||
p.populate_set_out(mssg, 'ModuleStats')
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
|
||||
#screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x'
|
||||
sleep 0.1
|
||||
|
|
@ -142,7 +142,7 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Dox" bash -c './Dox.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c './ApiKey.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
|
||||
sleep 0.1
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ subscribe = Redis_CurveManageTopSets
|
|||
|
||||
[Categ]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_Dox
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey
|
||||
|
||||
[CreditCards]
|
||||
subscribe = Redis_CreditCards
|
||||
|
|
@ -76,7 +76,7 @@ publish = Redis_alertHandler,Redis_Duplicate
|
|||
|
||||
[Dox]
|
||||
subscribe = Redis_Dox
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_ModuleStats
|
||||
publish = Redis_Duplicate,Redis_alertHandler
|
||||
|
||||
[ModuleStats]
|
||||
subscribe = Redis_ModuleStats
|
||||
|
|
@ -110,6 +110,10 @@ publish = Redis_Duplicate,Redis_alertHandler
|
|||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler
|
||||
|
||||
[ApiKey]
|
||||
subscribe = Redis_ApiKey
|
||||
publish = Redis_Duplicate,Redis_alertHandler
|
||||
|
||||
[Base64]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
amazon
|
||||
amazonaws
|
||||
amzn
|
||||
aws
|
||||
googleapis
|
||||
|
|
@ -87,6 +87,7 @@
|
|||
<li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#apikey-tab" data-attribute-name="apikey" data-panel="apikey-panel">API Keys</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#onion-tab" data-attribute-name="onion" data-panel="onion-panel">Onions</a></li>
|
||||
|
|
@ -112,6 +113,9 @@
|
|||
<div class="col-lg-12 tab-pane fade" id="keys-tab">
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="col-lg-12 tab-pane fade" id="apikey-tab">
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="col-lg-12 tab-pane fade" id="mail-tab">
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Reference in New Issue