add apiKeys module

pull/201/head
Terrtia 2018-04-26 14:42:39 +02:00
parent 29c368250e
commit c3428df5dc
7 changed files with 127 additions and 6 deletions

103
bin/ApiKey.py Executable file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python3.5
# -*-coding:UTF-8 -*
"""
The ApiKey Module
======================
This module is consuming the Redis-list created by the Categ module.
It apply API_key regexes on paste content and warn if above a threshold.
"""
import redis
import pprint
import time
import re
from packages import Paste
from packages import lib_refine
from pubsublogger import publisher
from Helper import Process
def search_api_key(message):
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
aws_access_key = regex_aws_access_key.findall(content)
aws_secret_key = regex_aws_secret_key.findall(content)
google_api_key = regex_google_api_key.findall(content)
print(aws_access_key)
print(aws_secret_key)
print(google_api_key)
if(len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0):
print('-------------------------------')
print(aws_access_key)
print(aws_secret_key)
print(google_api_key)
to_print = 'ApiKey;{};{};{};'.format(
paste.p_source, paste.p_date, paste.p_name)
if(len(google_api_key) > 0):
print('found google api key')
print(to_print)
publisher.warning('{}Checked {} found Google API Key;{}'.format(
to_print, len(google_api_key), paste.p_path))
if(len(aws_access_key) > 0 or len(aws_secret_key) > 0):
print('found AWS key')
print(to_print)
total = len(aws_access_key) + len(aws_secret_key)
publisher.warning('{}Checked {} found AWS Key;{}'.format(
to_print, total, paste.p_path))
msg = 'apikey;{}'.format(filename)
p.populate_set_out(msg, 'alertHandler')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'ApiKey'
p = Process(config_section)
publisher.info("ApiKey started")
# REDIS #
r_serv2 = redis.StrictRedis(
host=p.config.get("Redis_Cache", "host"),
port=p.config.getint("Redis_Cache", "port"),
db=p.config.getint("Redis_Cache", "db"))
message = p.get_from_set()
# TODO improve REGEX
regex_aws_access_key = re.compile(r'(?<![A-Z0-9])=[A-Z0-9]{20}(?![A-Z0-9])')
regex_aws_secret_key = re.compile(r'(?<!=[A-Za-z0-9+])=[A-Za-z0-9+]{40}(?![A-Za-z0-9+])')
regex_google_api_key = re.compile(r'=AIza[0-9a-zA-Z-_]{35}')
while True:
message = p.get_from_set()
if message is not None:
search_api_key(message)
else:
publisher.debug("Script ApiKey is Idling 10s")
#print('Sleeping')
time.sleep(10)

View File

@ -67,7 +67,7 @@ if __name__ == "__main__":
# FUNCTIONS # # FUNCTIONS #
publisher.info("Script Categ started") publisher.info("Script Categ started")
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'Dox'] categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'ApiKey']
tmp_dict = {} tmp_dict = {}
for filename in categories: for filename in categories:
bname = os.path.basename(filename) bname = os.path.basename(filename)

View File

@ -118,7 +118,7 @@ if __name__ == "__main__":
site_occurence = re.findall(regex_site_for_stats, content) site_occurence = re.findall(regex_site_for_stats, content)
for site in site_occurence: for site in site_occurence:
site_domain = site[1:-1] site_domain = site[1:-1]
if site_domain in creds_sites.keys(): if site_domain.encode('utf8') in creds_sites.keys():
creds_sites[site_domain] += 1 creds_sites[site_domain] += 1
else: else:
creds_sites[site_domain] = 1 creds_sites[site_domain] = 1
@ -132,6 +132,11 @@ if __name__ == "__main__":
creds_sites[domain] = 1 creds_sites[domain] = 1
for site, num in creds_sites.items(): # Send for each different site to moduleStats for site, num in creds_sites.items(): # Send for each different site to moduleStats
try:
site = site.decode('utf8')
except:
pass
mssg = 'credential;{};{};{}'.format(num, site, paste.p_date) mssg = 'credential;{};{};{}'.format(num, site, paste.p_date)
print(mssg) print(mssg)
p.populate_set_out(mssg, 'ModuleStats') p.populate_set_out(mssg, 'ModuleStats')

View File

@ -130,7 +130,7 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x' screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' #screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x' screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x'
sleep 0.1 sleep 0.1
@ -142,7 +142,7 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x' screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Dox" bash -c './Dox.py; read x' screen -S "Script_AIL" -X screen -t "ApiKey" bash -c './ApiKey.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x' screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
sleep 0.1 sleep 0.1

View File

@ -45,7 +45,7 @@ subscribe = Redis_CurveManageTopSets
[Categ] [Categ]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_Dox publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey
[CreditCards] [CreditCards]
subscribe = Redis_CreditCards subscribe = Redis_CreditCards
@ -76,7 +76,7 @@ publish = Redis_alertHandler,Redis_Duplicate
[Dox] [Dox]
subscribe = Redis_Dox subscribe = Redis_Dox
publish = Redis_Duplicate,Redis_alertHandler,Redis_ModuleStats publish = Redis_Duplicate,Redis_alertHandler
[ModuleStats] [ModuleStats]
subscribe = Redis_ModuleStats subscribe = Redis_ModuleStats
@ -110,6 +110,10 @@ publish = Redis_Duplicate,Redis_alertHandler
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Duplicate,Redis_alertHandler publish = Redis_Duplicate,Redis_alertHandler
[ApiKey]
subscribe = Redis_ApiKey
publish = Redis_Duplicate,Redis_alertHandler
[Base64] [Base64]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Duplicate,Redis_alertHandler publish = Redis_Duplicate,Redis_alertHandler

5
files/ApiKey Normal file
View File

@ -0,0 +1,5 @@
amazon
amazonaws
amzn
aws
googleapis

View File

@ -87,6 +87,7 @@
<li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li> <li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li> <li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li> <li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#apikey-tab" data-attribute-name="apikey" data-panel="apikey-panel">API Keys</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li> <li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li> <li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#onion-tab" data-attribute-name="onion" data-panel="onion-panel">Onions</a></li> <li name='nav-pan'><a data-toggle="tab" href="#onion-tab" data-attribute-name="onion" data-panel="onion-panel">Onions</a></li>
@ -112,6 +113,9 @@
<div class="col-lg-12 tab-pane fade" id="keys-tab"> <div class="col-lg-12 tab-pane fade" id="keys-tab">
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;"> <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
</div> </div>
<div class="col-lg-12 tab-pane fade" id="apikey-tab">
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
</div>
<div class="col-lg-12 tab-pane fade" id="mail-tab"> <div class="col-lg-12 tab-pane fade" id="mail-tab">
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;"> <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
</div> </div>