CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #164 from mokaddem/showDiff 

Show differences between duplicates
pull/181/head
Alexandre Dulaunoy 2018-02-21 22:25:16 +01:00 committed by GitHub
parent 0263b25da8 c39e537cbe
commit abdf41fe23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 32 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/Mixer.py
View File

@ -20,6 +20,7 @@ Depending on the configuration, this module will process the feed as follow:
- Else, do not process it but keep track for statistics on duplicate - Else, do not process it but keep track for statistics on duplicate
operation_mode 3: "Don't look if duplicated content" operation_mode 3: "Don't look if duplicated content"
- SImply do not bother to check if it is a duplicate
- Simply do not bother to check if it is a duplicate - Simply do not bother to check if it is a duplicate
Note that the hash of the content is defined as the sha1(gzip64encoded). Note that the hash of the content is defined as the sha1(gzip64encoded).

8
bin/packages/config.cfg.sample
View File

@ -21,15 +21,17 @@ sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon
##### Flask ##### ##### Flask #####
[Flask] [Flask]
#Maximum number of character to display in the toolip #Maximum number of character to display in the toolip
max_preview_char = 250 max_preview_char = 250
#Maximum number of character to display in the modal #Maximum number of character to display in the modal
max_preview_modal = 800 max_preview_modal = 800
#Default number of header to display in trending graphs #Default number of header to display in trending graphs
default_display = 10 default_display = 10
#Number of minutes displayed for the number of processed pastes. #Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10 minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
#### Modules #### #### Modules ####
[Categ] [Categ]
#Minimum number of match between the paste and the category file #Minimum number of match between the paste and the category file
matchingThreshold=1 matchingThreshold=1

1
var/www/modules/Flask_config.py
View File

@ -63,3 +63,4 @@ max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value

20
var/www/modules/showpaste/Flask_showpaste.py
View File

@ -7,7 +7,8 @@
import redis import redis
import json import json
import flask import flask
from flask import Flask, render_template, jsonify, request, Blueprint from flask import Flask, render_template, jsonify, request, Blueprint, make_response
import difflib
import Paste import Paste
@ -20,6 +21,7 @@ r_serv_pasteName = Flask_config.r_serv_pasteName
max_preview_char = Flask_config.max_preview_char max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal max_preview_modal = Flask_config.max_preview_modal
tlsh_to_percent = Flask_config.tlsh_to_percent tlsh_to_percent = Flask_config.tlsh_to_percent
DiffMaxLineLength = Flask_config.DiffMaxLineLength
showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates') showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
@ -113,5 +115,21 @@ def getmoredata():
to_return = p_content[max_preview_modal-1:] to_return = p_content[max_preview_modal-1:]
return to_return return to_return
@showsavedpastes.route("/showDiff/")
def showDiff():
s1 = request.args.get('s1', '')
s2 = request.args.get('s2', '')
p1 = Paste.Paste(s1)
p2 = Paste.Paste(s2)
maxLengthLine1 = p1.get_lines_info()[1]
maxLengthLine2 = p2.get_lines_info()[1]
if maxLengthLine1 > DiffMaxLineLength or maxLengthLine2 > DiffMaxLineLength:
return "Can't make the difference as the lines are too long."
htmlD = difflib.HtmlDiff()
lines1 = p1.get_p_content().decode('utf8', 'ignore').splitlines()
lines2 = p2.get_p_content().decode('utf8', 'ignore').splitlines()
the_html = htmlD.make_file(lines1, lines2)
return the_html
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(showsavedpastes) app.register_blueprint(showsavedpastes)

10
var/www/modules/showpaste/templates/show_saved_paste.html
View File

@ -15,14 +15,14 @@
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
</head> </head>
<body> <body>
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1> <h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1>
<h2 class="page-header" >({{ request.args.get('num') }})</h2> <h2 class="page-header" >({{ request.args.get('num') }})</h2>
<table class="table table-condensed"> <table class="table table-condensed">
<thead> <thead>
<tr> <tr>
@ -62,7 +62,8 @@
<th>Hash type</th> <th>Hash type</th>
<th>Paste info</th> <th>Paste info</th>
<th>Date</th> <th>Date</th>
<th>Path</th> <th>Path</th>
<th>Action</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -72,6 +73,7 @@
<td>Similarity: {{ simil_list[i] }}%</td> <td>Similarity: {{ simil_list[i] }}%</td>
<td>{{ date_list[i] }}</td> <td>{{ date_list[i] }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td> <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showDiff') }}?s1={{ request.args.get('paste') }}&s2={{ dup_path }}" class="fa fa-columns" title="Show differences"></a></td>
</tr> </tr>
{% set i = i + 1 %} {% set i = i + 1 %}
{% endfor %} {% endfor %}
@ -82,7 +84,7 @@
<p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p> <p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
</div> </div>
</div> </div>
</body> </body>
<script> <script>
$('#tableDup').DataTable(); $('#tableDup').DataTable();