CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'AntoniaBK-categories'

pull/926/head
Raphaël Vinot 2024-07-16 14:28:33 +02:00
parent b7332acb93 8808e203ce
commit ff8ec51ef2
6 changed files with 183 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lookyloo/lookyloo.py
View File

@ -306,7 +306,8 @@ class Lookyloo():
if categ_file.exists():
with categ_file.open() as f:
current_categories = [line.strip() for line in f.readlines()]
return {e: self.taxonomies.revert_machinetag(e) for e in current_categories}
# return {e: self.taxonomies.revert_machinetag(e) for e in current_categories}
return {e: e for e in current_categories}
return {}
def categorize_capture(self, capture_uuid: str, /, category: str) -> None:
@ -314,7 +315,7 @@ class Lookyloo():
if not get_config('generic', 'enable_categorization'):
return
# Make sure the category is mappable to a taxonomy.
self.taxonomies.revert_machinetag(category)
# self.taxonomies.revert_machinetag(category)
categ_file = self._captures_index[capture_uuid].capture_dir / 'categories'
# get existing categories if possible
@ -338,9 +339,10 @@ class Lookyloo():
current_categories = {line.strip() for line in f.readlines()}
else:
current_categories = set()
current_categories.remove(category)
with categ_file.open('w') as f:
f.writelines(f'{t}\n' for t in current_categories)
if category in current_categories:
current_categories.remove(category)
with categ_file.open('w') as f:
f.writelines(f'{t}\n' for t in current_categories)
def trigger_modules(self, capture_uuid: str, /, force: bool=False, auto_trigger: bool=False) -> dict[str, Any]:
'''Launch the 3rd party modules on a capture.

29
website/web/__init__.py
View File

@ -659,19 +659,36 @@ def historical_lookups(tree_uuid: str) -> str | WerkzeugResponse | Response:
circl_pdns=data.get('circl_pdns'))
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''}, methods=['GET', 'POST'])
@app.route('/tree/<string:tree_uuid>/categories_capture/<string:query>', methods=['GET'])
@flask_login.login_required # type: ignore[misc]
def categories_capture(tree_uuid: str, query: str) -> str | WerkzeugResponse | Response:
if not enable_categorization:
return redirect(url_for('tree', tree_uuid=tree_uuid))
current_categories = lookyloo.categories_capture(tree_uuid)
matching_categories = None
if query:
if 'verification-status' in request.form:
status = request.form.get('verification-status')
# fast categories
categories = []
possible_ctgs = {
'legitimate': ["parking-page", "default-page", 'institution', 'captcha', 'authentication-form', 'adult-content', 'shop'],
'malicious': ['clone', 'phishing', 'captcha', 'authentication-form', 'adult-content', 'shop'],
'unclear': ['captcha', 'authentication-form', 'adult-content', 'shop']
}
if status in possible_ctgs.keys():
lookyloo.categorize_capture(tree_uuid, status)
for category in possible_ctgs[status]:
if category in request.form:
categories.append(category)
for category in categories:
lookyloo.categorize_capture(tree_uuid, category)
if 'query' in request.form and request.form.get('query', '').strip():
matching_categories = {}
t = get_taxonomies()
entries = t.search(query)
if entries:
matching_categories = {e: t.revert_machinetag(e) for e in entries}
current_categories = lookyloo.categories_capture(tree_uuid)
return render_template('categories_capture.html', tree_uuid=tree_uuid,
current_categories=current_categories,
matching_categories=matching_categories)
@ -679,20 +696,22 @@ def categories_capture(tree_uuid: str, query: str) -> str | WerkzeugResponse | R
@app.route('/tree/<string:tree_uuid>/uncategorize/', defaults={'category': ''})
@app.route('/tree/<string:tree_uuid>/uncategorize/<string:category>', methods=['GET'])
@flask_login.login_required # type: ignore[misc]
def uncategorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse | Response:
if not enable_categorization:
return jsonify({'response': 'Categorization not enabled.'})
lookyloo.uncategorize_capture(tree_uuid, category)
return jsonify({'response': f'{category} successfully added to {tree_uuid}'})
return jsonify({'response': f'{category} successfully removed from {tree_uuid}'})
@app.route('/tree/<string:tree_uuid>/categorize/', defaults={'category': ''})
@app.route('/tree/<string:tree_uuid>/categorize/<string:category>', methods=['GET'])
@flask_login.login_required # type: ignore[misc]
def categorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse | Response:
if not enable_categorization:
return jsonify({'response': 'Categorization not enabled.'})
lookyloo.categorize_capture(tree_uuid, category)
return jsonify({'response': f'{category} successfully removed from {tree_uuid}'})
return jsonify({'response': f'{category} successfully added to {tree_uuid}'})
@app.route('/tree/<string:tree_uuid>/stats', methods=['GET'])

2
website/web/sri.txt
View File

@ -35,7 +35,7 @@
"secure.svg": "H8ni7t0d60nCJDVGuZpuxC+RBy/ipAjWT627D12HlZGg6LUmjSwPTQTUekm3UJupEP7TUkhXyq6WHc5gy7QBjg==",
"stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==",
"stats_graph.js": "S/sMNQK1UMMLD0xQeEa7sq3ce8o6oPxwxGlyKVtaHOODjair86dbBDm7cu6pa/elMRDJT1j09jEFjWp+5GbhTw==",
"tree.css": "xffZ5VGbH0dvaD3pJRj48PttTd29xtU45QozhHi/oJCKy8HU/NkHcGutpx9sOGMuY5tsKn8Pub6Ncsjj9f4TSg==",
"tree.css": "jc7+RiJaZy7utfMu7iMWicpt0y0ZFiEQlB4c7MFNdlWcZf0czi3LgSQUFlDWt828Mx463V+JP1RalXuRjbGcEg==",
"tree.js": "Czw6AAUPQsIawEXi+eM9WG4vqfBw/y9vKbn0EedI3QG5+Y5ryZF05kGEexRc04wj5ec8RNRbR6QGnaVX3DAq3g==",
"up.jpg": "d1ljZJ9f5JekyM6RLFFH2Ua44j6neiQBdUIXOenRTjGppQr3JaeglpQIH6BjPCJL177+TH52U3UIRNS5YAyKIg==",
"up_right.jpg": "OMmz+n+MxR34P8/fn5t4DkqKqdJRzQbXQ7fAi2lhkZIJGhVs2vIyY1f2hpYoBxDAX1OcYsSE2lqIR2vXNDGZsA==",

14
website/web/static/tree.css
View File

@ -185,4 +185,16 @@ hr {
background: white;
border: 2px solid;
padding: 5px;
}
}
/* Fast categories*/
.fast-categories {
border: 2px solid;
margin: 5px;
padding: 5px;
}
.fast-categories fieldset {
border: 1px solid;
padding: 5px;
}

4
website/web/templates/macros.html
View File

@ -49,12 +49,12 @@
$('.categorize_capture').on('click',function(e){
var button = $(this);
$.get("{{ url_for('categorize_capture', tree_uuid=tree_uuid) }}" + button.val())
$('.modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}")
$('#categoriesModal .modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}")
});
$('.uncategorize_capture').on('click',function(e){
var button = $(this);
$.get("{{ url_for('uncategorize_capture', tree_uuid=tree_uuid) }}" + button.val())
$('.modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}")
$('#categoriesModal .modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}")
});
</script>
{% endmacro %}

143
website/web/templates/tree.html
View File

@ -67,13 +67,77 @@
var modal = $(this);
modal.find('.modal-body').load(button.data("remote"));
});
$('#searchCategories').submit(function(event){
/*$('#searchCategories').submit(function(event){
var query = $("#query").val();
$('.modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}" + query, function() {
$('#categoriesModal .modal-body').load("{ url_for('categories_capture', tree_uuid=tree_uuid) }}" + query, function() {
$('#categoriesModal').modal({show:true});
});
event.preventDefault();
});
*/
$('#fast-categories').submit(function(event){
event.preventDefault();
var query = $('#fast-categories').serialize(); // Serialize form data
$.ajax({
type: 'POST',
url: '{{ url_for('categories_capture', tree_uuid=tree_uuid) }}',
data: query,
success: function(response) {
$('#categoriesModal .modal-body').html(response);
$('#categoriesModal').modal({show:true});
},
error: function(error) {
console.log("An error occurred: ", error);
}
});
});
// Fast categories
document.getElementById("legitimate").addEventListener('change', function(){
if (this.checked){
disableInput(document.getElementById('malicious-categories'));
enableInput(document.getElementById('legitimate-categories'));
}
});
document.getElementById("malicious").addEventListener('change', function(){
if (this.checked){
enableInput(document.getElementById('malicious-categories'));
disableInput(document.getElementById('legitimate-categories'));
}
});
document.getElementById("unclear").addEventListener('change', function(){
if (this.checked){
disableInput(document.getElementById('malicious-categories'));
disableInput(document.getElementById('legitimate-categories'));
}
});
document.getElementById("legitimate-categories").addEventListener('click', function(){
if (this.querySelectorAll('input[type="checkbox"]:checked').length > 0) {
document.getElementById("legitimate").checked = true;
}
});
document.getElementById("malicious-categories").addEventListener('click', function(){
if (this.querySelectorAll('input[type="checkbox"]:checked').length > 0) {
document.getElementById("malicious").checked = true;
}
});
function disableInput(container) {
const Input = container.querySelectorAll('input');
Input.forEach(function(checkbox){
checkbox.disabled = true;
checkbox.checked = false;
});
container.hidden = true;
}
function enableInput(container) {
const Input = container.querySelectorAll('input');
Input.forEach(checkbox => checkbox.disabled = false);
container.hidden = false;
}
</script>
<script>
@ -420,7 +484,7 @@
<div id="extra-menu" class="dropdown">
<button class="dropbtn">Extras</button>
<div id="extra-menu-content" class="dropdown-content">
{% if enable_categorization %}
{% if enable_categorization and current_user.is_authenticated %}
<a href="#categoriesModal" data-remote="{{ url_for('categories_capture', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#categoriesModal" role="button">Manage categories</a>
{% endif %}
@ -1033,7 +1097,7 @@
</div>
</div>
{% if enable_context_by_users %}
{% if enable_categorization %}
<div class="modal fade" id="categoriesModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">
@ -1045,11 +1109,76 @@
... loading the categorization options ...
</div>
<p>
<form id=searchCategories>
<form role="form" id="fast-categories" class="fast-categories">
<fieldset id="verification-status">
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" id="legitimate" name="verification-status" value="legitimate">
<label class="form-check-label" for="legitimate">legitimate</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" id="malicious" name="verification-status" value="malicious">
<label class="form-check-label" for="malicious">malicious</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" id="unclear" name="verification-status" value="unclear" checked>
<label class="form-check-label" for="unclear">unclear</label>
</div>
</fieldset>
<div id="content-categoies">
<p>Content Categories (Multiple Choices possible)</p>
<fieldset id="legitimate-categories" hidden>
<p>Only if legitimate:</p>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="parking-page" name="parking-page" value="parking-page">
<label class="form-check-label" for="parking-page">parking-page</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="default-page" name="default-page" value="default-page">
<label class="form-check-label" for="default-page">default-page</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="institution" name="institution" value="institution">
<label class="form-check-label" for="institution">institution / company</label><br>
</div>
<!-- the name of the institution/company could be added here in the future, maybe with propositions
<label for="institution-name">Enter the name of the institution/company:</label><br>
<input type="text" id="institution-name" name="institution-name">
-->
</fieldset>
<fieldset id="malicious-categories" hidden>
<p>Only if malicious:</p>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="clone" name="clone" value="clone">
<label class="form-check-label" for="clone">clone</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="phishing" name="phishing" value="phishing">
<label class="form-check-label" for="phishing">phishing</label><br>
</div>
</fieldset>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="captcha" name="captcha" value="captcha">
<label class="form-check-label" for="captcha">captcha</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="authentication-form" name="authentication-form" value="authentication-form">
<label class="form-check-label" for="authentication-form">authentication-form</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="adult-content" name="adult-content" value="adult-content">
<label class="form-check-label" for="adult-content">adult-content</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="shop" name="shop" value="shop">
<label class="form-check-label" for="shop">shop</label><br>
</div>
</div>
<label for="query" class="form-label">Category to search</label>
<input type="text" class="form-control" name="query" id="query" placeholder="Query">
<button type="submit" class="btn btn-success">Search</button>
</form>
<button type="submit" class="btn btn-success" name="submit">Submit</button>
</form>
</p>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>