From 754ae9354d49245dd0ca677009e2560717258108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 11 May 2020 19:01:02 +0200 Subject: [PATCH] new: Add email notification feature --- config/email.tmpl | 10 +++++ config/generic.json.sample | 13 +++++- lookyloo/helpers.py | 5 +++ lookyloo/lookyloo.py | 70 ++++++++++++++++++++------------- poetry.lock | 27 ++++++------- website/web/__init__.py | 13 +++++- website/web/templates/tree.html | 29 ++++++++++++++ 7 files changed, 122 insertions(+), 45 deletions(-) create mode 100644 config/email.tmpl diff --git a/config/email.tmpl b/config/email.tmpl new file mode 100644 index 00000000..325fec70 --- /dev/null +++ b/config/email.tmpl @@ -0,0 +1,10 @@ +Dear {recipient}, + +Please have a look at this capture on lookyloo: + * https://{domain}/tree/{uuid} + +{comment} + + +Best regards, +{sender} diff --git a/config/generic.json.sample b/config/generic.json.sample index b4ab9ea4..cb3b215c 100644 --- a/config/generic.json.sample +++ b/config/generic.json.sample @@ -9,12 +9,23 @@ "days": 1, "hours": 0 }, + "enable_mail_notification": false, + "email": { + "from": "Lookyloo ", + "to": "Investigation Team ", + "subject": "Capture from Lookyloo to review", + "domain": "lookyloo.myorg.local", + "smtp_host": "localhost", + "smtp_port": "25" + }, "_notes": { "loglevel": "(lookyloo) Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels", "splash_loglevel": "(Splash) INFO is *very* verbose.", "only_global_lookups": "Set it to True if your instance is publicly available so users aren't able to scan your internal network", "splash_url": "URL to connect to splash", "cache_clean_user": "Format: {username: password}", - "time_delta_on_index": "Time interval of the capture displayed on the index" + "time_delta_on_index": "Time interval of the capture displayed on the index", + "enable_mail_notification": "Enable email notification or not", + "email": "Configuration for sending email notifications." } } diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index e0afc27c..ccefb0cd 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -41,6 +41,11 @@ Run the following command (assuming you run the code from the clonned repository return Path(os.environ['LOOKYLOO_HOME']) +def get_email_template(): + with (get_homedir() / 'config' / 'email.tmpl').open() as f: + return f.read() + + def load_configs(path_to_config_files: Optional[Union[str, Path]]=None) -> Dict[str, Dict[str, Any]]: if path_to_config_files: if isinstance(path_to_config_files, str): diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index c4471bf4..a3db163a 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -1,39 +1,29 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import json - -import pickle - -from datetime import datetime - -import tempfile -import pathlib -import time - -import ipaddress -import socket -from urllib.parse import urlsplit - -from io import BufferedIOBase, BytesIO import base64 +from datetime import datetime +from email.message import EmailMessage +from io import BufferedIOBase, BytesIO +import ipaddress +import json +import logging +from pathlib import Path +import pickle +import smtplib +import socket +from typing import Union, Dict, List, Tuple, Optional, Any +from urllib.parse import urlsplit from uuid import uuid4 -from pathlib import Path -from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir -from .exceptions import NoValidHarFile -from redis import Redis - -from typing import Union, Dict, List, Tuple, Optional, Any - -import logging - -from pysanejs import SaneJS -from scrapysplashwrapper import crawl -from har2tree import CrawledTree, Har2TreeError, HarFile - from defang import refang # type: ignore +from har2tree import CrawledTree, Har2TreeError, HarFile +from pysanejs import SaneJS +from redis import Redis +from scrapysplashwrapper import crawl +from .exceptions import NoValidHarFile +from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template from .modules import VirusTotal @@ -245,6 +235,30 @@ class Lookyloo(): return pickle.load(_p) return None + def send_mail(self, capture_uuid: str, comment: str=''): + if not self.get_config('enable_mail_notification'): + return + email_config = self.get_config('email') + msg = EmailMessage() + msg['From'] = email_config['from'] + msg['To'] = email_config['to'] + msg['Subject'] = email_config['subject'] + body = get_email_template() + body = body.format( + recipient=msg['To'].addresses[0].display_name, + domain=email_config['domain'], + uuid=capture_uuid, + comment=comment, + sender=msg['From'].addresses[0].display_name, + ) + msg.set_content(body) + try: + s = smtplib.SMTP(email_config['smtp_host'], email_config['smtp_port']) + s.send_message(msg) + s.quit() + except Exception as e: + logging.exception(e) + def load_tree(self, capture_dir: Path) -> Tuple[str, dict, str, str, str, dict]: har_files = sorted(capture_dir.glob('*.har')) pickle_file = capture_dir / 'tree.pickle' diff --git a/poetry.lock b/poetry.lock index 39720f3e..e3b29022 100644 --- a/poetry.lock +++ b/poetry.lock @@ -514,17 +514,14 @@ description = "Parsel is a library to extract data from HTML and XML using XPath name = "parsel" optional = false python-versions = "*" -version = "1.5.2" +version = "1.6.0" [package.dependencies] cssselect = ">=0.9" -six = ">=1.5.2" +lxml = "*" +six = ">=1.6.0" w3lib = ">=1.19.0" -[package.dependencies.lxml] -python = "<3.4.0 || >=3.5.0" -version = "*" - [[package]] category = "dev" description = "A Python Parser" @@ -742,7 +739,7 @@ description = "pytest: simple powerful testing with Python" name = "pytest" optional = false python-versions = ">=3.5" -version = "5.4.1" +version = "5.4.2" [package.dependencies] atomicwrites = ">=1.0" @@ -791,7 +788,7 @@ description = "Python client for Redis key-value store" name = "redis" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "3.5.0" +version = "3.5.1" [package.extras] hiredis = ["hiredis (>=0.1.3)"] @@ -873,7 +870,7 @@ scrapy = "^1.8.0" scrapy-splash = "^0.7.2" [package.source] -reference = "61a0a67290fd7d2ee3353aad0b03fa125ded1a5d" +reference = "a67b329d4b4281a90418c2e63464523294af6d53" type = "git" url = "https://github.com/viper-framework/ScrapySplashWrapper.git" [[package]] @@ -1407,8 +1404,8 @@ packaging = [ {file = "packaging-20.3.tar.gz", hash = "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3"}, ] parsel = [ - {file = "parsel-1.5.2-py2.py3-none-any.whl", hash = "sha256:74f8e9d3b345b14cb1416bd777a03982cde33a74d8b32e0c71e651d07d41d40a"}, - {file = "parsel-1.5.2.tar.gz", hash = "sha256:4da4262ba4605573b6b72a5f557616a2fc9dee7a47a1efad562752a28d366723"}, + {file = "parsel-1.6.0-py2.py3-none-any.whl", hash = "sha256:9e1fa8db1c0b4a878bf34b35c043d89c9d1cbebc23b4d34dbc3c0ec33f2e087d"}, + {file = "parsel-1.6.0.tar.gz", hash = "sha256:70efef0b651a996cceebc69e55a85eb2233be0890959203ba7c3a03c72725c79"}, ] parso = [ {file = "parso-0.7.0-py2.py3-none-any.whl", hash = "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0"}, @@ -1505,8 +1502,8 @@ pypydispatcher = [ ] pysanejs = [] pytest = [ - {file = "pytest-5.4.1-py3-none-any.whl", hash = "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172"}, - {file = "pytest-5.4.1.tar.gz", hash = "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970"}, + {file = "pytest-5.4.2-py3-none-any.whl", hash = "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3"}, + {file = "pytest-5.4.2.tar.gz", hash = "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698"}, ] pytest-httpserver = [ {file = "pytest_httpserver-0.3.4-py3-none-any.whl", hash = "sha256:7feab352b2626d1a0ecdebffcac5e5875979f08ad7e621b2289980ce8f6ebc5b"}, @@ -1517,8 +1514,8 @@ queuelib = [ {file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"}, ] redis = [ - {file = "redis-3.5.0-py2.py3-none-any.whl", hash = "sha256:174101a3ce04560d716616290bb40e0a2af45d5844c8bd474c23fc5c52e7a46a"}, - {file = "redis-3.5.0.tar.gz", hash = "sha256:7378105cd8ea20c4edc49f028581e830c01ad5f00be851def0f4bc616a83cd89"}, + {file = "redis-3.5.1-py2.py3-none-any.whl", hash = "sha256:a5b0e25890d216d8189636742c50ab992e42eea699bcc1b08cc2d6bf3adff52a"}, + {file = "redis-3.5.1.tar.gz", hash = "sha256:6e9d2722a95d10ddf854596e66516d316d99c6a483e5db3b35c34e1158b2bfa1"}, ] requests = [ {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, diff --git a/website/web/__init__.py b/website/web/__init__.py index ae97bac6..f324d9fd 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -235,6 +235,13 @@ def cache_tree(tree_uuid): return redirect(url_for('index')) +@app.route('/tree//send_mail', methods=['POST']) +def send_mail(tree_uuid): + comment = request.form.get('comment') if request.form.get('comment') else '' + lookyloo.send_mail(tree_uuid, comment) + return redirect(url_for('tree', tree_uuid=tree_uuid)) + + @app.route('/tree/', methods=['GET']) def tree(tree_uuid): if tree_uuid == 'False': @@ -251,10 +258,14 @@ def tree(tree_uuid): return redirect(url_for('index')) try: + if lookyloo.get_config('enable_mail_notification'): + enable_mail_notification = True + else: + enable_mail_notification = False tree_json, start_time, user_agent, root_url, meta = load_tree(capture_dir) return render_template('tree.html', tree_json=tree_json, start_time=start_time, user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid, - meta=meta) + meta=meta, enable_mail_notification=enable_mail_notification) except NoValidHarFile as e: return render_template('error.html', error_message=e) diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html index 24305c82..c7604d84 100644 --- a/website/web/templates/tree.html +++ b/website/web/templates/tree.html @@ -140,6 +140,9 @@ Show third party reports + {% if enable_mail_notification %} + Notify by mail + {%endif%}
@@ -173,4 +176,30 @@
+ + + {% endblock content %}