CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'AntoniaBK-categories'

pull/926/head
Raphaël Vinot 2024-07-16 14:28:33 +02:00
parent b7332acb93 8808e203ce
commit ff8ec51ef2
6 changed files with 183 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lookyloo/lookyloo.py
View File

@ -306,7 +306,8 @@ class Lookyloo():
if categ_file.exists(): if categ_file.exists():
with categ_file.open() as f: with categ_file.open() as f:
current_categories = [line.strip() for line in f.readlines()] current_categories = [line.strip() for line in f.readlines()]
return {e: self.taxonomies.revert_machinetag(e) for e in current_categories} # return {e: self.taxonomies.revert_machinetag(e) for e in current_categories}
return {e: e for e in current_categories}
return {} return {}
def categorize_capture(self, capture_uuid: str, /, category: str) -> None: def categorize_capture(self, capture_uuid: str, /, category: str) -> None:
@ -314,7 +315,7 @@ class Lookyloo():
if not get_config('generic', 'enable_categorization'): if not get_config('generic', 'enable_categorization'):
return return
# Make sure the category is mappable to a taxonomy. # Make sure the category is mappable to a taxonomy.
self.taxonomies.revert_machinetag(category) # self.taxonomies.revert_machinetag(category)
categ_file = self._captures_index[capture_uuid].capture_dir / 'categories' categ_file = self._captures_index[capture_uuid].capture_dir / 'categories'
# get existing categories if possible # get existing categories if possible
@ -338,9 +339,10 @@ class Lookyloo():
current_categories = {line.strip() for line in f.readlines()} current_categories = {line.strip() for line in f.readlines()}
else: else:
current_categories = set() current_categories = set()
current_categories.remove(category) if category in current_categories:
with categ_file.open('w') as f: current_categories.remove(category)
f.writelines(f'{t}\n' for t in current_categories) with categ_file.open('w') as f:
f.writelines(f'{t}\n' for t in current_categories)
def trigger_modules(self, capture_uuid: str, /, force: bool=False, auto_trigger: bool=False) -> dict[str, Any]: def trigger_modules(self, capture_uuid: str, /, force: bool=False, auto_trigger: bool=False) -> dict[str, Any]:
'''Launch the 3rd party modules on a capture. '''Launch the 3rd party modules on a capture.

29
website/web/__init__.py
View File

@ -659,19 +659,36 @@ def historical_lookups(tree_uuid: str) -> str | WerkzeugResponse | Response:
circl_pdns=data.get('circl_pdns')) circl_pdns=data.get('circl_pdns'))
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''}) @app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''}, methods=['GET', 'POST'])
@app.route('/tree/<string:tree_uuid>/categories_capture/<string:query>', methods=['GET']) @app.route('/tree/<string:tree_uuid>/categories_capture/<string:query>', methods=['GET'])
@flask_login.login_required # type: ignore[misc]
def categories_capture(tree_uuid: str, query: str) -> str | WerkzeugResponse | Response: def categories_capture(tree_uuid: str, query: str) -> str | WerkzeugResponse | Response:
if not enable_categorization: if not enable_categorization:
return redirect(url_for('tree', tree_uuid=tree_uuid)) return redirect(url_for('tree', tree_uuid=tree_uuid))
current_categories = lookyloo.categories_capture(tree_uuid)
matching_categories = None matching_categories = None
if query: if 'verification-status' in request.form:
status = request.form.get('verification-status')
# fast categories
categories = []
possible_ctgs = {
'legitimate': ["parking-page", "default-page", 'institution', 'captcha', 'authentication-form', 'adult-content', 'shop'],
'malicious': ['clone', 'phishing', 'captcha', 'authentication-form', 'adult-content', 'shop'],
'unclear': ['captcha', 'authentication-form', 'adult-content', 'shop']
}
if status in possible_ctgs.keys():
lookyloo.categorize_capture(tree_uuid, status)
for category in possible_ctgs[status]:
if category in request.form:
categories.append(category)
for category in categories:
lookyloo.categorize_capture(tree_uuid, category)
if 'query' in request.form and request.form.get('query', '').strip():
matching_categories = {} matching_categories = {}
t = get_taxonomies() t = get_taxonomies()
entries = t.search(query) entries = t.search(query)
if entries: if entries:
matching_categories = {e: t.revert_machinetag(e) for e in entries} matching_categories = {e: t.revert_machinetag(e) for e in entries}
current_categories = lookyloo.categories_capture(tree_uuid)
return render_template('categories_capture.html', tree_uuid=tree_uuid, return render_template('categories_capture.html', tree_uuid=tree_uuid,
current_categories=current_categories, current_categories=current_categories,
matching_categories=matching_categories) matching_categories=matching_categories)
@ -679,20 +696,22 @@ def categories_capture(tree_uuid: str, query: str) -> str | WerkzeugResponse | R
@app.route('/tree/<string:tree_uuid>/uncategorize/', defaults={'category': ''}) @app.route('/tree/<string:tree_uuid>/uncategorize/', defaults={'category': ''})
@app.route('/tree/<string:tree_uuid>/uncategorize/<string:category>', methods=['GET']) @app.route('/tree/<string:tree_uuid>/uncategorize/<string:category>', methods=['GET'])
@flask_login.login_required # type: ignore[misc]
def uncategorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse | Response: def uncategorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse | Response:
if not enable_categorization: if not enable_categorization:
return jsonify({'response': 'Categorization not enabled.'}) return jsonify({'response': 'Categorization not enabled.'})
lookyloo.uncategorize_capture(tree_uuid, category) lookyloo.uncategorize_capture(tree_uuid, category)
return jsonify({'response': f'{category} successfully added to {tree_uuid}'}) return jsonify({'response': f'{category} successfully removed from {tree_uuid}'})
@app.route('/tree/<string:tree_uuid>/categorize/', defaults={'category': ''}) @app.route('/tree/<string:tree_uuid>/categorize/', defaults={'category': ''})
@app.route('/tree/<string:tree_uuid>/categorize/<string:category>', methods=['GET']) @app.route('/tree/<string:tree_uuid>/categorize/<string:category>', methods=['GET'])
@flask_login.login_required # type: ignore[misc]
def categorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse | Response: def categorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse | Response:
if not enable_categorization: if not enable_categorization:
return jsonify({'response': 'Categorization not enabled.'}) return jsonify({'response': 'Categorization not enabled.'})
lookyloo.categorize_capture(tree_uuid, category) lookyloo.categorize_capture(tree_uuid, category)
return jsonify({'response': f'{category} successfully removed from {tree_uuid}'}) return jsonify({'response': f'{category} successfully added to {tree_uuid}'})
@app.route('/tree/<string:tree_uuid>/stats', methods=['GET']) @app.route('/tree/<string:tree_uuid>/stats', methods=['GET'])

2
website/web/sri.txt
View File

@ -35,7 +35,7 @@
"secure.svg": "H8ni7t0d60nCJDVGuZpuxC+RBy/ipAjWT627D12HlZGg6LUmjSwPTQTUekm3UJupEP7TUkhXyq6WHc5gy7QBjg==", "secure.svg": "H8ni7t0d60nCJDVGuZpuxC+RBy/ipAjWT627D12HlZGg6LUmjSwPTQTUekm3UJupEP7TUkhXyq6WHc5gy7QBjg==",
"stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==", "stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==",
"stats_graph.js": "S/sMNQK1UMMLD0xQeEa7sq3ce8o6oPxwxGlyKVtaHOODjair86dbBDm7cu6pa/elMRDJT1j09jEFjWp+5GbhTw==", "stats_graph.js": "S/sMNQK1UMMLD0xQeEa7sq3ce8o6oPxwxGlyKVtaHOODjair86dbBDm7cu6pa/elMRDJT1j09jEFjWp+5GbhTw==",
"tree.css": "xffZ5VGbH0dvaD3pJRj48PttTd29xtU45QozhHi/oJCKy8HU/NkHcGutpx9sOGMuY5tsKn8Pub6Ncsjj9f4TSg==", "tree.css": "jc7+RiJaZy7utfMu7iMWicpt0y0ZFiEQlB4c7MFNdlWcZf0czi3LgSQUFlDWt828Mx463V+JP1RalXuRjbGcEg==",
"tree.js": "Czw6AAUPQsIawEXi+eM9WG4vqfBw/y9vKbn0EedI3QG5+Y5ryZF05kGEexRc04wj5ec8RNRbR6QGnaVX3DAq3g==", "tree.js": "Czw6AAUPQsIawEXi+eM9WG4vqfBw/y9vKbn0EedI3QG5+Y5ryZF05kGEexRc04wj5ec8RNRbR6QGnaVX3DAq3g==",
"up.jpg": "d1ljZJ9f5JekyM6RLFFH2Ua44j6neiQBdUIXOenRTjGppQr3JaeglpQIH6BjPCJL177+TH52U3UIRNS5YAyKIg==", "up.jpg": "d1ljZJ9f5JekyM6RLFFH2Ua44j6neiQBdUIXOenRTjGppQr3JaeglpQIH6BjPCJL177+TH52U3UIRNS5YAyKIg==",
"up_right.jpg": "OMmz+n+MxR34P8/fn5t4DkqKqdJRzQbXQ7fAi2lhkZIJGhVs2vIyY1f2hpYoBxDAX1OcYsSE2lqIR2vXNDGZsA==", "up_right.jpg": "OMmz+n+MxR34P8/fn5t4DkqKqdJRzQbXQ7fAi2lhkZIJGhVs2vIyY1f2hpYoBxDAX1OcYsSE2lqIR2vXNDGZsA==",

12
website/web/static/tree.css
View File

@ -186,3 +186,15 @@ hr {
border: 2px solid; border: 2px solid;
padding: 5px; padding: 5px;
} }
/* Fast categories*/
.fast-categories {
border: 2px solid;
margin: 5px;
padding: 5px;
}
.fast-categories fieldset {
border: 1px solid;
padding: 5px;
}

4
website/web/templates/macros.html
View File

@ -49,12 +49,12 @@
$('.categorize_capture').on('click',function(e){ $('.categorize_capture').on('click',function(e){
var button = $(this); var button = $(this);
$.get("{{ url_for('categorize_capture', tree_uuid=tree_uuid) }}" + button.val()) $.get("{{ url_for('categorize_capture', tree_uuid=tree_uuid) }}" + button.val())
$('.modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}") $('#categoriesModal .modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}")
}); });
$('.uncategorize_capture').on('click',function(e){ $('.uncategorize_capture').on('click',function(e){
var button = $(this); var button = $(this);
$.get("{{ url_for('uncategorize_capture', tree_uuid=tree_uuid) }}" + button.val()) $.get("{{ url_for('uncategorize_capture', tree_uuid=tree_uuid) }}" + button.val())
$('.modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}") $('#categoriesModal .modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}")
}); });
</script> </script>
{% endmacro %} {% endmacro %}

143
website/web/templates/tree.html
View File

@ -67,13 +67,77 @@
var modal = $(this); var modal = $(this);
modal.find('.modal-body').load(button.data("remote")); modal.find('.modal-body').load(button.data("remote"));
}); });
$('#searchCategories').submit(function(event){ /*$('#searchCategories').submit(function(event){
var query = $("#query").val(); var query = $("#query").val();
$('.modal-body').load("{{ url_for('categories_capture', tree_uuid=tree_uuid) }}" + query, function() { $('#categoriesModal .modal-body').load("{ url_for('categories_capture', tree_uuid=tree_uuid) }}" + query, function() {
$('#categoriesModal').modal({show:true}); $('#categoriesModal').modal({show:true});
}); });
event.preventDefault(); event.preventDefault();
}); });
*/
$('#fast-categories').submit(function(event){
event.preventDefault();
var query = $('#fast-categories').serialize(); // Serialize form data
$.ajax({
type: 'POST',
url: '{{ url_for('categories_capture', tree_uuid=tree_uuid) }}',
data: query,
success: function(response) {
$('#categoriesModal .modal-body').html(response);
$('#categoriesModal').modal({show:true});
},
error: function(error) {
console.log("An error occurred: ", error);
}
});
});
// Fast categories
document.getElementById("legitimate").addEventListener('change', function(){
if (this.checked){
disableInput(document.getElementById('malicious-categories'));
enableInput(document.getElementById('legitimate-categories'));
}
});
document.getElementById("malicious").addEventListener('change', function(){
if (this.checked){
enableInput(document.getElementById('malicious-categories'));
disableInput(document.getElementById('legitimate-categories'));
}
});
document.getElementById("unclear").addEventListener('change', function(){
if (this.checked){
disableInput(document.getElementById('malicious-categories'));
disableInput(document.getElementById('legitimate-categories'));
}
});
document.getElementById("legitimate-categories").addEventListener('click', function(){
if (this.querySelectorAll('input[type="checkbox"]:checked').length > 0) {
document.getElementById("legitimate").checked = true;
}
});
document.getElementById("malicious-categories").addEventListener('click', function(){
if (this.querySelectorAll('input[type="checkbox"]:checked').length > 0) {
document.getElementById("malicious").checked = true;
}
});
function disableInput(container) {
const Input = container.querySelectorAll('input');
Input.forEach(function(checkbox){
checkbox.disabled = true;
checkbox.checked = false;
});
container.hidden = true;
}
function enableInput(container) {
const Input = container.querySelectorAll('input');
Input.forEach(checkbox => checkbox.disabled = false);
container.hidden = false;
}
</script> </script>
<script> <script>
@ -420,7 +484,7 @@
<div id="extra-menu" class="dropdown"> <div id="extra-menu" class="dropdown">
<button class="dropbtn">Extras</button> <button class="dropbtn">Extras</button>
<div id="extra-menu-content" class="dropdown-content"> <div id="extra-menu-content" class="dropdown-content">
{% if enable_categorization %} {% if enable_categorization and current_user.is_authenticated %}
<a href="#categoriesModal" data-remote="{{ url_for('categories_capture', tree_uuid=tree_uuid) }}" <a href="#categoriesModal" data-remote="{{ url_for('categories_capture', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#categoriesModal" role="button">Manage categories</a> data-bs-toggle="modal" data-bs-target="#categoriesModal" role="button">Manage categories</a>
{% endif %} {% endif %}
@ -1033,7 +1097,7 @@
</div> </div>
</div> </div>
{% if enable_context_by_users %} {% if enable_categorization %}
<div class="modal fade" id="categoriesModal" tabindex="-1" role="dialog"> <div class="modal fade" id="categoriesModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document"> <div class="modal-dialog modal-xl" role="document">
<div class="modal-content"> <div class="modal-content">
@ -1045,11 +1109,76 @@
... loading the categorization options ... ... loading the categorization options ...
</div> </div>
<p> <p>
<form id=searchCategories> <form role="form" id="fast-categories" class="fast-categories">
<fieldset id="verification-status">
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" id="legitimate" name="verification-status" value="legitimate">
<label class="form-check-label" for="legitimate">legitimate</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" id="malicious" name="verification-status" value="malicious">
<label class="form-check-label" for="malicious">malicious</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" id="unclear" name="verification-status" value="unclear" checked>
<label class="form-check-label" for="unclear">unclear</label>
</div>
</fieldset>
<div id="content-categoies">
<p>Content Categories (Multiple Choices possible)</p>
<fieldset id="legitimate-categories" hidden>
<p>Only if legitimate:</p>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="parking-page" name="parking-page" value="parking-page">
<label class="form-check-label" for="parking-page">parking-page</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="default-page" name="default-page" value="default-page">
<label class="form-check-label" for="default-page">default-page</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="institution" name="institution" value="institution">
<label class="form-check-label" for="institution">institution / company</label><br>
</div>
<!-- the name of the institution/company could be added here in the future, maybe with propositions
<label for="institution-name">Enter the name of the institution/company:</label><br>
<input type="text" id="institution-name" name="institution-name">
-->
</fieldset>
<fieldset id="malicious-categories" hidden>
<p>Only if malicious:</p>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="clone" name="clone" value="clone">
<label class="form-check-label" for="clone">clone</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="phishing" name="phishing" value="phishing">
<label class="form-check-label" for="phishing">phishing</label><br>
</div>
</fieldset>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="captcha" name="captcha" value="captcha">
<label class="form-check-label" for="captcha">captcha</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="authentication-form" name="authentication-form" value="authentication-form">
<label class="form-check-label" for="authentication-form">authentication-form</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="adult-content" name="adult-content" value="adult-content">
<label class="form-check-label" for="adult-content">adult-content</label><br>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="shop" name="shop" value="shop">
<label class="form-check-label" for="shop">shop</label><br>
</div>
</div>
<label for="query" class="form-label">Category to search</label> <label for="query" class="form-label">Category to search</label>
<input type="text" class="form-control" name="query" id="query" placeholder="Query"> <input type="text" class="form-control" name="query" id="query" placeholder="Query">
<button type="submit" class="btn btn-success">Search</button> <button type="submit" class="btn btn-success" name="submit">Submit</button>
</form> </form>
</p> </p>
<div class="modal-footer"> <div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button> <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>