CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'website-improvement' into production

pull/94/head
Mokaddem 2016-11-25 16:01:40 +01:00
parent 5ad77d4d47 5d269ea1ee
commit 0cabd5c4fe
18 changed files with 542 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/Credential.py
View File

@ -48,7 +48,7 @@ if __name__ == "__main__":
if sites_set:
message += ' Related websites: {}'.format(', '.join(sites_set))
to_print = 'Credential;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message)
to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_path)
print('\n '.join(creds))

6
bin/CreditCards.py
View File

@ -63,14 +63,14 @@ if __name__ == "__main__":
to_print = 'CreditCard;{};{};{};'.format(
paste.p_source, paste.p_date, paste.p_name)
if (len(creditcard_set) > 0):
publisher.warning('{}Checked {} valid number(s)'.format(
to_print, len(creditcard_set)))
publisher.warning('{}Checked {} valid number(s);{}'.format(
to_print, len(creditcard_set), paste.p_path))
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
else:
publisher.info('{}CreditCard related'.format(to_print))
publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path))
else:
publisher.debug("Script creditcard is idling 1m")
time.sleep(10)

8
bin/DomClassifier.py
View File

@ -51,13 +51,13 @@ def main():
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld))
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc))
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
except IOError:
print "CRC Checksum Failed on :", PST.p_path
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(

9
bin/Duplicates.py
View File

@ -131,8 +131,10 @@ if __name__ == "__main__":
# index of paste
index_current = r_serv_dico.get(dico_hash)
paste_path = r_serv_dico.get(index_current)
paste_date = r_serv_dico.get(index_current+'_date')
paste_date = paste_date if paste_date != None else "No date available"
if paste_path != None:
hash_dico[dico_hash] = (hash_type, paste_path, percent)
hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
print '['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
except Exception,e:
@ -142,6 +144,7 @@ if __name__ == "__main__":
# Add paste in DB after checking to prevent its analysis twice
# hash_type_i -> index_i AND index_i -> PST.PATH
r_serv1.set(index, PST.p_path)
r_serv1.set(index+'_date', PST._get_p_date())
r_serv1.sadd("INDEX", index)
# Adding hashes in Redis
for hash_type, paste_hash in paste_hashes.iteritems():
@ -152,7 +155,7 @@ if __name__ == "__main__":
# if there is data in this dictionnary
if len(hash_dico) != 0:
# paste_tuple = (paste_path, percent)
# paste_tuple = (hash_type, date, paste_path, percent)
for dico_hash, paste_tuple in hash_dico.items():
dupl.append(paste_tuple)
@ -162,7 +165,7 @@ if __name__ == "__main__":
if dupl != []:
PST.__setattr__("p_duplicate", dupl)
PST.save_attribute_redis("p_duplicate", dupl)
publisher.info('{}Detected {}'.format(to_print, len(dupl)))
publisher.info('{}Detected {};{}'.format(to_print, len(dupl), PST.p_path))
print '{}Detected {}'.format(to_print, len(dupl))
y = time.time()

4
bin/Mail.py
View File

@ -55,9 +55,9 @@ if __name__ == "__main__":
list(MX_values[1])))
pprint.pprint(MX_values)
to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
to_print = 'Mails;{};{};{};Checked {} e-mail(s);{}'.\
format(PST.p_source, PST.p_date, PST.p_name,
MX_values[0])
MX_values[0], PST.p_path)
if MX_values[0] > is_critical:
publisher.warning(to_print)
#Send to duplicate

8
bin/Onion.py
View File

@ -133,8 +133,8 @@ if __name__ == "__main__":
PST.p_name)
if len(domains_list) > 0:
publisher.warning('{}Detected {} .onion(s)'.format(
to_print, len(domains_list)))
publisher.warning('{}Detected {} .onion(s);{}'.format(
to_print, len(domains_list),PST.p_path))
now = datetime.datetime.now()
path = os.path.join('onions', str(now.year).zfill(4),
str(now.month).zfill(2),
@ -144,9 +144,9 @@ if __name__ == "__main__":
PST.p_date,
PST.p_name)
for url in fetch(p, r_cache, urls, domains_list, path):
publisher.warning('{}Checked {}'.format(to_print, url))
publisher.warning('{}Checked {};{}'.format(to_print, url, PST.p_path))
else:
publisher.info('{}Onion related'.format(to_print))
publisher.info('{}Onion related;{}'.format(to_print, PST.p_path))
prec_filename = filename
else:

2
bin/Release.py
View File

@ -34,7 +34,7 @@ if __name__ == "__main__":
if len(releases) == 0:
continue
to_print = 'Release;{};{};{};{} releases'.format(paste.p_source, paste.p_date, paste.p_name, len(releases))
to_print = 'Release;{};{};{};{} releases;{}'.format(paste.p_source, paste.p_date, paste.p_name, len(releases), paste.p_path)
if len(releases) > 30:
publisher.warning(to_print)
else:

2
bin/SQLInjectionDetection.py
View File

@ -69,7 +69,7 @@ def analyse(url, path):
if (result_path > 1) or (result_query > 1):
print "Detected SQL in URL: "
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL")
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')

4
bin/Web.py
View File

@ -131,8 +131,8 @@ if __name__ == "__main__":
list(A_values[1])))
pprint.pprint(A_values)
publisher.info('Url;{};{};{};Checked {} URL'.format(
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
publisher.info('Url;{};{};{};Checked {} URL;{}'.format(
PST.p_source, PST.p_date, PST.p_name, A_values[0], PST.p_path))
prec_filename = filename
else:

114
var/www/Flask_server.py
View File

@ -78,6 +78,27 @@ def event_stream():
if msg['type'] == 'pmessage' and level != "DEBUG":
yield 'data: %s\n\n' % json.dumps(msg)
def event_stream_getImportantPasteByModule(module_name):
index = 0
all_pastes_list = getPastebyType(r_serv_db, module_name)
for path in all_pastes_list:
index += 1
paste = Paste.Paste(path)
content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
data = {}
data["module"] = module_name
data["index"] = index
data["path"] = path
data["content"] = content[0:content_range]
data["linenum"] = paste.get_lines_info()[0]
data["date"] = curr_date
data["char_to_display"] = max_preview_modal
data["finished"] = True if index == len(all_pastes_list) else False
yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser
def get_queues(r):
# We may want to put the llen in a pipeline to do only one query.
@ -122,6 +143,7 @@ def showpaste(content_range):
p_duplicate_full_list = json.loads(paste._get_p_duplicate())
p_duplicate_list = []
p_simil_list = []
p_date_list = []
p_hashtype_list = []
@ -149,20 +171,29 @@ def showpaste(content_range):
hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types)
comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals)
new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals])
if len(p_duplicate_full_list[dup_list_index]) > 3:
try:
date_paste = str(int(p_duplicate_full_list[dup_list_index][3]))
date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8]
except ValueError:
date_paste = str(p_duplicate_full_list[dup_list_index][3])
else:
date_paste = "No date available"
new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste])
# Create the list to pass to the webpage
for dup_list in new_dup_list:
hash_type, path, simil_percent = dup_list
hash_type, path, simil_percent, date_paste = dup_list
p_duplicate_list.append(path)
p_simil_list.append(simil_percent)
p_hashtype_list.append(hash_type)
p_date_list.append(date_paste)
if content_range != 0:
p_content = p_content[0:content_range]
return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list)
return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
def getPastebyType(server, module_name):
all_path = []
@ -326,11 +357,10 @@ def search():
c = [] #preview of the paste content
paste_date = []
paste_size = []
num_elem_to_get = 50
# Search filename
print r_serv_pasteName.smembers(q[0])
for path in r_serv_pasteName.smembers(q[0]):
print path
r.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content().decode('utf8', 'ignore')
@ -351,7 +381,7 @@ def search():
from whoosh.qparser import QueryParser
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(" ".join(q))
results = searcher.search(query, limit=None)
results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
for x in results:
r.append(x.items()[0][1])
paste = Paste.Paste(x.items()[0][1])
@ -362,7 +392,57 @@ def search():
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_size.append(paste._get_p_size())
return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal)
results = searcher.search(query)
num_res = len(results)
return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res)
@app.route("/get_more_search_result", methods=['POST'])
def get_more_search_result():
query = request.form['query']
q = []
q.append(query)
page_offset = int(request.form['page_offset'])
num_elem_to_get = 50
path_array = []
preview_array = []
date_array = []
size_array = []
from whoosh import index
from whoosh.fields import Schema, TEXT, ID
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
ix = index.open_dir(indexpath)
from whoosh.qparser import QueryParser
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(" ".join(q))
results = searcher.search_page(query, page_offset, num_elem_to_get)
for x in results:
path_array.append(x.items()[0][1])
paste = Paste.Paste(x.items()[0][1])
content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
preview_array.append(content[0:content_range])
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
date_array.append(curr_date)
size_array.append(paste._get_p_size())
to_return = {}
to_return["path_array"] = path_array
to_return["preview_array"] = preview_array
to_return["date_array"] = date_array
to_return["size_array"] = size_array
print "len(path_array)="+str(len(path_array))
if len(path_array) < num_elem_to_get: #pagelength
to_return["moreData"] = False
else:
to_return["moreData"] = True
return jsonify(to_return)
@app.route("/")
@ -408,19 +488,30 @@ def importantPasteByModule():
paste_date = []
paste_linenum = []
all_path = []
allPastes = getPastebyType(r_serv_db, module_name)
for path in getPastebyType(r_serv_db, module_name):
for path in allPastes[0:10]:
all_path.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
all_content.append(content[0:content_range])
all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_linenum.append(paste.get_lines_info()[0])
return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal)
if len(allPastes) > 10:
finished = ""
else:
finished = "display: none;"
return render_template("important_paste_by_module.html", moduleName=module_name, all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished)
@app.route("/_getImportantPasteByModule")
def getImportantPasteByModule():
module_name = request.args.get('moduleName')
return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
@app.route("/moduletrending/")
def moduletrending():
@ -735,7 +826,8 @@ def showsavedpaste():
@app.route("/showpreviewpaste/")
def showpreviewpaste():
return showpaste(max_preview_modal)
num = request.args.get('num', '')
return "|num|"+num+"|num|"+showpaste(max_preview_modal)
@app.route("/getmoredata/")

71
var/www/static/js/indexjavascript.js
View File

@ -109,11 +109,11 @@ function create_log_table(obj_json) {
var pdate = document.createElement('TD')
var nam = document.createElement('TD')
var msage = document.createElement('TD')
var inspect = document.createElement('TD')
var chansplit = obj_json.channel.split('.');
var parsedmess = obj_json.data.split(';');
if (parsedmess[0] == "Global"){
var paste_processed = parsedmess[4].split(" ")[2];
window.paste_num_tabvar = paste_processed;
@ -139,7 +139,7 @@ function create_log_table(obj_json) {
source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
}
source_link.setAttribute("HREF",source_url);
source_link.setAttribute("TARGET", "_blank")
source_link.setAttribute("TARGET", "_blank");
source_link.appendChild(document.createTextNode(parsedmess[1]));
src.appendChild(source_link);
@ -169,6 +169,18 @@ function create_log_table(obj_json) {
msage.appendChild(document.createTextNode(message.join(" ")));
var paste_path = parsedmess[5];
var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
var action_icon_a = document.createElement("A");
action_icon_a.setAttribute("TARGET", "_blank");
action_icon_a.setAttribute("HREF", url_to_saved_paste);
var action_icon_span = document.createElement('SPAN');
action_icon_span.className = "fa fa-search-plus";
action_icon_a.appendChild(action_icon_span);
inspect.appendChild(action_icon_a);
tr.appendChild(time)
tr.appendChild(chan);
tr.appendChild(level);
@ -177,6 +189,7 @@ function create_log_table(obj_json) {
tr.appendChild(pdate);
tr.appendChild(nam);
tr.appendChild(msage);
tr.appendChild(inspect);
if (tr.className == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked == true) {
tableBody.appendChild(tr);
@ -219,28 +232,42 @@ function create_queue_table() {
tr.appendChild(th);
}
for(i = 0; i < (glob_tabvar.row1).length;i++){
var tr = document.createElement('TR')
for(j = 0; j < 2; j++){
var td = document.createElement('TD')
var moduleNum = j == 0 ? "." + glob_tabvar.row1[i][3] : "";
td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum));
tr.appendChild(td)
}
// Used to decide the color of the row
// We have glob_tabvar.row1[][j] with:
// - j=0: ModuleName
// - j=1: queueLength
// - j=2: LastProcessedPasteTime
// - j=3: Number of the module belonging in the same category
if (parseInt(glob_tabvar.row1[i][2]) > 60*2 && parseInt(glob_tabvar.row1[i][1]) > 2)
tr.className += " danger";
else if (parseInt(glob_tabvar.row1[i][2]) > 60*1)
tr.className += " warning";
else
tr.className += " success";
if ((glob_tabvar.row1).length == 0) {
var tr = document.createElement('TR');
var td = document.createElement('TD');
var td2 = document.createElement('TD');
td.appendChild(document.createTextNode("No running queues"));
td2.appendChild(document.createTextNode("Or no feed"));
td.className += " danger";
td2.className += " danger";
tr.appendChild(td);
tr.appendChild(td2);
tableBody.appendChild(tr);
}
else {
for(i = 0; i < (glob_tabvar.row1).length;i++){
var tr = document.createElement('TR')
for(j = 0; j < 2; j++){
var td = document.createElement('TD')
var moduleNum = j == 0 ? "." + glob_tabvar.row1[i][3] : "";
td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum));
tr.appendChild(td)
}
// Used to decide the color of the row
// We have glob_tabvar.row1[][j] with:
// - j=0: ModuleName
// - j=1: queueLength
// - j=2: LastProcessedPasteTime
// - j=3: Number of the module belonging in the same category
if (parseInt(glob_tabvar.row1[i][2]) > 60*2 && parseInt(glob_tabvar.row1[i][1]) > 2)
tr.className += " danger";
else if (parseInt(glob_tabvar.row1[i][2]) > 60*1)
tr.className += " warning";
else
tr.className += " success";
tableBody.appendChild(tr);
}
}
Tablediv.appendChild(table);
}

3
var/www/templates/browse_important_paste.html
View File

@ -30,9 +30,6 @@
white-space:pre-wrap;
word-wrap:break-word;
}
.modal-backdrop.fade {
opacity: 0;
}
</style>
</head>

195
var/www/templates/important_paste_by_module.html
View File

@ -1,4 +1,4 @@
<table class="table table-striped table-bordered table-hover" id="myTable">
<table class="table table-striped table-bordered table-hover" id="myTable_{{ moduleName }}">
<thead>
<tr>
<th>#</th>
@ -23,12 +23,110 @@
</tbody>
</table>
</br>
<div id="nbr_entry" class="alert alert-info">
</div>
<div id="div_stil_data">
<button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries(100)" style="display: none">Load 100 entries</button>
<button id="load_more_json_button2" type="button" class="btn btn-warning" onclick="add_entries(300)" style="display: none">Load 300 entries</button>
<img id="loading_gif_browse" src="{{url_for('static', filename='image/loading.gif') }}" heigt="20" width="20" style="margin: 2px; {{ finished }}"></div>
</br>
<script>
var json_array = [];
var all_data_received = false;
var curr_numElem;
var elem_added = 0;
var tot_num_entry = 0;
function deploy_source() {
var button_load_more_displayed = false;
if(typeof(EventSource) !== "undefined" && typeof(source) !== "") {
var source = new EventSource("{{ url_for('getImportantPasteByModule') }}"+"?moduleName="+moduleName);
source.onmessage = function(event) {
var feed = jQuery.parseJSON( event.data );
curr_numElem = parseInt($("#myTable_"+moduleName).attr('data-numElem'));
if (feed.index > curr_numElem & feed.module == moduleName) { // Avoid doubling the pastes
json_array.push(feed);
tot_num_entry++;
$("#load_more_json_button1").show();
$("#load_more_json_button2").show();
$("#nbr_entry").text(tot_num_entry + " entries available, " + (tot_num_entry - elem_added) + " not displayed");
$("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
if(feed.index > 100 && !button_load_more_displayed) {
button_load_more_displayed = true;
add_entries_X(20);
}
if(feed.finished) {
$("#loading_gif_browse").hide();
source.close();
all_data_received = true;
add_entries_X(10);
}
}
};
} else {
console.log("Sorry, your browser does not support server-sent events...");
}
}
function add_entries(iter) { //Used to disable the button before going to the big loop
$("#load_more_json_button1").attr('disabled','disabled');
$("#load_more_json_button2").attr('disabled','disabled');
setTimeout(function() { add_entries_X(iter);}, 50);
}
function add_entries_X(to_add) {
for(i=0; i<to_add; i++) {
if(json_array.length == 0 && all_data_received) {
$("#load_more_json_button1").hide();
$("#load_more_json_button2").hide();
$("#nbr_entry").hide();
return false;
} else {
var feed = json_array.shift();
elem_added++;
search_table.row.add( [
feed.index,
"<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\"> "+ feed.path +"</a>",
feed.date,
feed.linenum,
"<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+feed.content.replace(/\"/g, "\'").replace(/\r/g, "\'").replace(/\n/g, "\'")+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+feed.index+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\" data-path=\""+feed.path+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
] ).draw( false );
$("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
}
}
$("#load_more_json_button1").removeAttr('disabled');
$("#load_more_json_button2").removeAttr('disabled');
$("#nbr_entry").text(tot_num_entry + " entries available, " + (tot_num_entry - elem_added) + " not displayed");
return true;
}
</script>
<script>
var moduleName = "{{ moduleName }}";
var search_table;
var last_clicked_paste;
var can_change_modal_content = true;
$("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}");
$(document).ready(function(){
$('[data-toggle="tooltip"]').tooltip();
$('#myTable').dataTable({ "order": [[ 2, "desc" ]] });
search_table = $('#myTable_'+moduleName).DataTable({ "order": [[ 2, "desc" ]] });
deploy_source();
});
</script>
@ -40,39 +138,9 @@ $(document).ready(function(){
var char_to_display = {{ char_to_display }};
var start_index = 0;
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click.openmodal", function (event) {
event.preventDefault();
var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
$.get(url, function (data) {
$("#mymodalbody").html(data);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
});
});
// When the modal goes out, refresh it to normal content
$("#mymodal").on('hidden.bs.modal', function () {
can_change_modal_content = true;
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
@ -110,38 +178,53 @@ $(document).ready(function(){
}
// Use to bind the button with the new displayed data
// (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
$('#myTable').on( 'draw.dt', function () {
search_table.on( 'draw.dt', function () {
// Bind tooltip each time we draw a new page
$('[data-toggle="tooltip"]').tooltip();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) {
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
event.preventDefault();
var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
last_clicked_paste = $(this).attr('data-num');
$.get(url, function (data) {
$("#mymodalbody").html(data);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
// verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
var received_num = parseInt(data.split("|num|")[1]);
if (received_num == last_clicked_paste && can_change_modal_content) {
can_change_modal_content = false;
// clear data by removing html, body, head tags. prevent dark modal background stack bug.
var cleared_data = data.split("<body>")[1].split("</body>")[0];
$("#mymodalbody").html(cleared_data);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
});
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
} else if (can_change_modal_content) {
$("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
}
});
});
} );

2
var/www/templates/index.html
View File

@ -140,6 +140,7 @@
<th>Date</th>
<th>Paste name</th>
<th>Message</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="tab_body">
@ -153,6 +154,7 @@
<!-- /#page-wrapper -->
</div>
<script> var url_showSavedPath = "{{ url_for('showsavedpaste') }}"; </script>
<script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"></script>
<script>

161
var/www/templates/search.html
View File

@ -31,9 +31,7 @@
white-space:pre-wrap;
word-wrap:break-word;
}
.modal-backdrop.fade {
opacity: 0;
}
</style>
</head>
@ -69,7 +67,7 @@
</div>
<div class="modal-footer">
<a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
@ -84,7 +82,7 @@
</br>
<div class="panel panel-default">
<div class="panel-heading">
<i class="glyphicon glyphicon-search"></i> {{ r|length }} Results for "<strong>{{ query }}</strong>"
<i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>
<div class="pull-right">
</div>
@ -101,7 +99,7 @@
<th>Action</th>
</tr>
</thead>
<tbody>
<tbody id="table_body">
{% set i = 0 %}
{% for path in r %}
<tr>
@ -115,7 +113,11 @@
{% endfor %}
</tbody>
</table>
<div id="div_stil_data">
<button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries();" style="display: none">Load 50 entries</button>
<strong> Totalling {{ num_res }} results related to paste content </strong>
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
@ -124,15 +126,78 @@
<!-- /#page-wrapper -->
</div>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</br>
</body>
<!-- enable tooltip and dataTable -->
<script>
var search_table;
var last_clicked_paste;
var can_change_modal_content = true;
var page_offset;
var offset;
var all_loaded;
var init_num_of_elements_in_table;
var query;
var pagelen = 50;
$(document).ready(function(){
$('[data-toggle="tooltip"]').tooltip();
$("#button_show_path").hide();
$('#myTable').dataTable();
search_table = $('#myTable').DataTable();
query = "{{ query }}";
offset = 0;
page_offset = 2; //page 1 already loaded
all_loaded = false;
init_num_of_elements_in_table = parseInt("{{ r|length }}"); // Comes from the file search
if (init_num_of_elements_in_table == pagelen) {
$("#load_more_json_button1").show();
}
});
</script>
<script>
// Loop to recover all the data from get_more_search_results
// And add it dynamically top the dataTable
function add_entries() { //Used to disable the button before going to the big loop
$("#load_more_json_button1").attr('disabled','disabled');
setTimeout(function() { load_search_50_data();}, 50);
}
function load_search_50_data() {
var options = { query: query, page_offset: page_offset };
$.post( "{{ url_for('get_more_search_result') }}", options).done(function( data ) {
for(i=0; i<data.path_array.length; i++) {
var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
search_table.row.add( [
init_num_of_elements_in_table+((offset))+i+1,
"<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\"> "+ data.path_array[i] +"</a>",
data.date_array[i],
data.size_array[i],
"<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+curr_preview+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+i+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\" data-path=\""+data.path_array[i]+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
] ).draw( false );
}
offset = offset + data.path_array.length;
page_offset = page_offset+1;
$("#numberOfRes").text(parseInt($("#numberOfRes").text()) + data.path_array.length);
if (data.moreData == true) {
//continue
} else {
all_loaded = true;
$("#load_more_json_button1").hide();
}
$("#load_more_json_button1").removeAttr('disabled');
return data.path_array.length;
});
}
</script>
<!-- Dynamically update the modal -->
@ -143,39 +208,9 @@
var char_to_display = {{ char_to_display }};
var start_index = 0;
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click", function (event) {
event.preventDefault();
var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
$.get(url, function (data) {
$("#mymodalbody").html(data);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
});
});
// When the modal goes out, refresh it to normal content
$("#mymodal").on('hidden.bs.modal', function () {
can_change_modal_content = true;
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
@ -211,5 +246,55 @@
new_content.show('fast');
$("#load-more-button").hide();
}
$('#myTable').on( 'draw.dt', function () {
// Bind tooltip each time we draw a new page
$('[data-toggle="tooltip"]').tooltip();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
last_clicked_paste = $(this).attr('data-num');
$.get(url, function (data) {
// verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
var received_num = parseInt(data.split("|num|")[1]);
if (received_num == last_clicked_paste && can_change_modal_content) {
can_change_modal_content = false;
// clear data by removing html, body, head tags. prevent dark modal background stack bug.
var cleared_data = data.split("<body>")[1].split("</body>")[0];
$("#mymodalbody").html(cleared_data);
setTimeout(function() { $('#tableDup').DataTable(); }, 150);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// On click, donwload all paste's content
$("#load-more-button").off('click.download').on("click.download", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
} else if (can_change_modal_content) {
$("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
}
});
});
} );
</script>
</html>

20
var/www/templates/sentiment_analysis_trending.html
View File

@ -205,24 +205,24 @@
</thead>
<tbody>
<tr>
<td class="worst1 moodtable_worst">worst1</td>
<td class="best1 moodtable_best">best1</td>
<td class="worst1 moodtable_worst">no data</td>
<td class="best1 moodtable_best">no data</td>
</tr>
<tr>
<td class="worst2 moodtable_worst">worst2</td>
<td class="best2 moodtable_best">best2</td>
<td class="worst2 moodtable_worst">no data</td>
<td class="best2 moodtable_best">no data</td>
</tr>
<tr>
<td class="worst3 moodtable_worst">worst3</td>
<td class="best3 moodtable_best">best3</td>
<td class="worst3 moodtable_worst">no data</td>
<td class="best3 moodtable_best">no data</td>
</tr>
<tr>
<td class="worst4 moodtable_worst">worst4</td>
<td class="best4 moodtable_best">best4</td>
<td class="worst4 moodtable_worst">no data</td>
<td class="best4 moodtable_best">no data</td>
</tr>
<tr>
<td class="worst5 moodtable_worst">worst5</td>
<td class="best5 moodtable_best">best5</td>
<td class="worst5 moodtable_worst">no data</td>
<td class="best5 moodtable_best">no data</td>
</tr>
</tbody>
</table>

133
var/www/templates/show_saved_paste.html
View File

@ -4,68 +4,87 @@
<title>Paste information</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
</head>
<body>
<div class="panel panel-default">
<div class="panel-heading">
<h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1>
<h2 class="page-header" >({{ request.args.get('num') }})</h2>
<h2> Paste: {{ request.args.get('num') }}</h2>
<h3> {{ request.args.get('paste') }} </h3>
<hr></br>
<div class="panel panel-default">
<div class="panel-heading">
<table class="table table-condensed">
<thead>
<tr>
<th>Date</th>
<th>Source</th>
<th>Encoding</th>
<th>Language</th>
<th>Size (Kb)</th>
<th>Mime</th>
<th>Number of lines</th>
<th>Max line length</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ date }}</td>
<td>{{ source }}</td>
<td>{{ encoding }}</td>
<td>{{ language }}</td>
<td>{{ size }}</td>
<td>{{ mime }}</td>
<td>{{ lineinfo.0 }}</td>
<td>{{ lineinfo.1 }}</td>
</tr>
</tbody>
</table>
</div>
<div class="panel-body" id="panel-body">
{% if duplicate_list|length == 0 %}
<h3> No Duplicate </h3>
{% else %}
<h3> Duplicate list: </h3>
<table style="width:100%">
{% set i = 0 %}
<tr>
<th style="text-align:left;">Hash type</th><th style="text-align:left;">Paste info</th>
</tr>
{% for dup_path in duplicate_list %}
<table class="table table-condensed">
<thead>
<tr>
<th>Date</th>
<th>Source</th>
<th>Encoding</th>
<th>Language</th>
<th>Size (Kb)</th>
<th>Mime</th>
<th>Number of lines</th>
<th>Max line length</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ date }}</td>
<td>{{ source }}</td>
<td>{{ encoding }}</td>
<td>{{ language }}</td>
<td>{{ size }}</td>
<td>{{ mime }}</td>
<td>{{ lineinfo.0 }}</td>
<td>{{ lineinfo.1 }}</td>
</tr>
</tbody>
</table>
</div>
<div class="panel-body" id="panel-body">
{% if duplicate_list|length == 0 %}
<h3> No Duplicate </h3>
{% else %}
<h3> Duplicate list: </h3>
<table id="tableDup" class="table table-striped table-bordered">
{% set i = 0 %}
<thead>
<tr>
<td>{{ hashtype_list[i] }}</td>
<td>Similarity: {{ simil_list[i] }}%</td>
<td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
<th>Hash type</th>
<th>Paste info</th>
<th>Date</th>
<th>Path</th>
</tr>
{% set i = i + 1 %}
{% endfor %}
</table>
{% endif %}
<h3> Content: </h3>
<p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
</div>
</div>
</thead>
<tbody>
{% for dup_path in duplicate_list %}
<tr>
<td>{{ hashtype_list[i] }}</td>
<td>Similarity: {{ simil_list[i] }}%</td>
<td>{{ date_list[i] }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
</tr>
{% set i = i + 1 %}
{% endfor %}
</tbody>
</table>
{% endif %}
<h3> Content: </h3>
<p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
</div>
</div>
</body>
<script>
$('#tableDup').DataTable();
</script>
</html>

15
var/www/templates/terms_management.html
View File

@ -111,7 +111,7 @@
<th>Day occurence</th>
<th>Week occurence</th>
<th>Month occurence</th>
<th># Concerned pastes</th>
<th># tracked paste</th>
<th>Action</th>
</tr>
</thead>
@ -198,6 +198,9 @@
<!-- import graph function -->
<script>
var table_track;
var table_black;
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
@ -207,6 +210,14 @@
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
table_track.on( 'draw.dt', function () {
perform_binding();
});
table_black.on( 'draw.dt', function () {
perform_binding();
});
$("#followTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
@ -254,7 +265,7 @@
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}