new: Basic support for monitoring

config/generic.json.sample

@@ -26,6 +26,10 @@
     "enable": false,
     "url": ""
   },
+  "monitoring": {
+    "enable": false,
+    "url": "https://127.0.0.1:5200"
+  },
   "tor_proxy": {
     "server": "socks5://127.0.0.1:9050"
   },
@@ -74,6 +78,7 @@
     "auto_trigger_modules": "Automatically trigger the modules when the tree is loaded and when the capture is cached",
     "enable_mail_notification": "Allow users to notify a pre-configured email address about a specific capture",
     "remote_lacus": "By default, lookyloo will do the capture locally. Enabling this feature means you have a dedicated Lacus instance somewhere",
+    "monitoring": "Enable connection to a remote monitoring instance",
     "tor_proxy": "URL to connect to a SOCKS 5 proxy for tor",
     "email": "Configuration for sending email notifications.",
     "priority": "Define the priority of a new capture. A capture from the web interface has priority over a capture from the API, same for authenticated user vs. anonymous.",

lookyloo/lookyloo.py

@@ -34,6 +34,7 @@ from pylacus import (PyLacus,
 #                     CaptureSettings as CaptureSettingsPy)
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 from pysecuritytxt import PySecurityTXT, SecurityTXTNotAvailable
+from pylookyloomonitoring import PyLookylooMonitoring
 from redis import ConnectionPool, Redis
 from redis.connection import UnixDomainSocketConnection
 
@@ -44,7 +45,8 @@ from .exceptions import (MissingCaptureDirectory,
                          MissingUUID, TreeNeedsRebuild, NoValidHarFile)
 from .helpers import (get_captures_dir, get_email_template,
                       get_resources_hashes, get_taxonomies,
-                      uniq_domains, ParsedUserAgent, load_cookies, UserAgents)
+                      uniq_domains, ParsedUserAgent, load_cookies, UserAgents,
+                      get_useragent_for_requests)
 from .indexing import Indexing
 from .modules import (MISP, PhishingInitiative, UniversalWhois,
                       UrlScan, VirusTotal, Phishtank, Hashlookup,
@@ -112,6 +114,14 @@ class Lookyloo():
         if not self.urlhaus.available:
             self.logger.warning('Unable to setup the URLhaus module')
 
+        self.monitoring_enabled = False
+        if monitoring_config := get_config('generic', 'monitoring'):
+            print(monitoring_config)
+            if monitoring_config['enable']:
+                self.monitoring = PyLookylooMonitoring(monitoring_config['url'], get_useragent_for_requests())
+                if self.monitoring.is_up:
+                    self.monitoring_enabled = True
+
         self.logger.info('Initializing context...')
         self.context = Context()
         self.logger.info('Context initialized.')

poetry.lock

@@ -2185,6 +2185,28 @@ requests = ">=2.28.2,<3.0.0"
 [package.extras]
 docs = ["Sphinx (>=6.1.3,<7.0.0)"]
 
+[[package]]
+name = "pylookyloomonitoring"
+version = "0.1.0"
+description = "Python API to connect to lookyloo monitoring"
+category = "main"
+optional = false
+python-versions = "^3.8"
+files = []
+develop = false
+
+[package.dependencies]
+requests = "^2.28.2"
+
+[package.extras]
+docs = ["Sphinx (>=6.1.3,<7.0.0)"]
+
+[package.source]
+type = "git"
+url = "https://github.com/Lookyloo/PyLookylooMonitoring.git"
+reference = "HEAD"
+resolved_reference = "99edb224dce7126c0ed8aabb4628e04d93e1580a"
+
 [[package]]
 name = "pymisp"
 version = "2.4.168"
@@ -3159,4 +3181,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.12"
-content-hash = "cb5328d141e3bafc70de74d7a715b842271da5c19c0fb4e9996907f3e5751959"
+content-hash = "37b91a1a0b9b1409a2206b80d029d5c5684850c56f839bb24b09d14b0f639c6f"

pyproject.toml

@@ -72,6 +72,7 @@ publicsuffixlist = "^0.9.3"
 pyfaup = "^1.2"
 chardet = "^5.1.0"
 pysecuritytxt = "^1.0.1"
+pylookyloomonitoring = {git = "https://github.com/Lookyloo/PyLookylooMonitoring.git"}
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.0.1"

website/web/__init__.py

@@ -621,6 +621,23 @@ def cache_tree(tree_uuid: str):
     return redirect(url_for('index'))
 
 
+@app.route('/tree/<string:tree_uuid>/monitor', methods=['POST', 'GET'])
+def monitor(tree_uuid: str):
+    if not lookyloo.monitoring_enabled:
+        return redirect(url_for('tree', tree_uuid=tree_uuid))
+    if request.form.get('name') or not request.form.get('confirm'):
+        # got a bot.
+        logging.info(f'{src_request_ip(request)} is a bot - {request.headers.get("User-Agent")}.')
+        return redirect('https://www.youtube.com/watch?v=iwGFalTRHDA')
+
+    collection: str = request.form['collection'] if request.form.get('collection') else ''
+    frequency: str = request.form['frequency'] if request.form.get('frequency') else 'daily'
+    cache = lookyloo.capture_cache(tree_uuid)
+    monitoring_uuid = lookyloo.monitoring.monitor({'url': cache.url}, frequency=frequency, collection=collection)
+    flash(f"Sent to monitoring: {monitoring_uuid}", 'success')
+    return redirect(url_for('tree', tree_uuid=tree_uuid))
+
+
 @app.route('/tree/<string:tree_uuid>/send_mail', methods=['POST', 'GET'])
 def send_mail(tree_uuid: str):
     if not enable_mail_notification:
@@ -688,6 +705,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
                                screenshot_thumbnail=b64_thumbnail, page_title=cache.title,
                                screenshot_size=screenshot_size,
                                meta=meta, enable_mail_notification=enable_mail_notification,
+                               enable_monitoring=lookyloo.monitoring_enabled,
                                enable_context_by_users=enable_context_by_users,
                                enable_categorization=enable_categorization,
                                enable_bookmark=enable_bookmark,

website/web/genericapi.py

@@ -14,6 +14,7 @@ from lacuscore import CaptureStatus as CaptureStatusCore
 from pylacus import CaptureStatus as CaptureStatusPy
 from lookyloo.lookyloo import Lookyloo
 from lookyloo.comparator import Comparator
+from lookyloo.exceptions import MissingUUID
 
 from .helpers import build_users_table, load_user_from_request, src_request_ip
 
@@ -455,7 +456,15 @@ class CompareCaptures(Resource):
     @api.doc(body=compare_captures_fields)
     def post(self):
         parameters: Dict = request.get_json(force=True)
-        result = comparator.compare_captures(parameters.get('capture_left'), parameters.get('capture_right'))
+        left_uuid = parameters.get('capture_left')
+        right_uuid = parameters.get('capture_right')
+        try:
+            result = comparator.compare_captures(left_uuid, right_uuid)
+        except MissingUUID as e:
+            # UUID non-existent, or capture still ongoing.
+            status_left = lookyloo.get_capture_status(left_uuid)
+            status_right = lookyloo.get_capture_status(right_uuid)
+            return {'error': e, 'details': {left_uuid: status_left, right_uuid: status_right}}
         return result
 
 

website/web/templates/tree.html

@@ -191,6 +191,12 @@
         <li class="list-group-item">
           <a href="{{ url_for('index') }}" role="button">Homepage</a>
         </li>
+        {% if enable_monitoring %}
+        <hr/>
+        <li class="list-group-item">
+          <a href="#monitoringModal" data-bs-toggle="modal" data-bs-target="#monitoringModal" role="button" class="btn btn-outline-info">Monitor</br>capture</a>
+        </li>
+        {% endif %}
         {% if enable_mail_notification %}
         <hr/>
         <li class="list-group-item">
@@ -681,6 +687,50 @@
 </div>
 {% endif %}
 
+{% if enable_monitoring %}
+<div class="modal fade" id="monitoringModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
+    <form role="form" action="{{ tree_uuid }}/monitor" method=post enctype=multipart/form-data>
+      <div class="modal-content">
+          <div class="modal-header">
+              <h5 class="modal-title" id="monitorModalLabel">Monitor capture</h5>
+              <button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+          </div>
+          <div class="modal-body">
+            <p>The capture will be submitted to the monitoring interface.</p>
+            <div class="mb-3">
+              <label for=frequency" class="form-label">Frequency</label>
+              <select name="frequency" id="frequency" class="form-select" aria-label="Select a frequency for the monitoring">
+                <option value="hourly" selected>Hourly</option>
+                <option value="daily">Daily</option>
+              </select>
+
+              <!-- boat fields -->
+              <label class="boatymcboat form-label" for="name">Your Name</label>
+              <input class="boatymcboat" autocomplete="off" type="text" id="name" name="name"
+                     placeholder="Your fav boat name here">
+            </div>
+            <div class="mb-3">
+              <label for="collection" class="form-label">Collection</label>
+              <input type="text" class="form-control" name="collection" id="collection" placeholder="Name of the collection">
+            </div>
+            <div class="mb-3 form-check">
+                <input class="form-check-input" type="checkbox" name="confirm" onchange="document.getElementById('btn-notification').disabled = !this.checked;"></input>
+                <label for="force_push" class="form-check-label">{{ confirm_message }}</label>
+            </div>
+          </div>
+          <div class="modal-footer">
+              <button type="submit" class="btn btn-success" id="btn-notification" disabled=true>Send to monitoring</button>
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+          </div>
+      </div>
+    </form>
+  </div>
+</div>
+{% endif %}
+
+<div class="modal fade" id="urlsInPageModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
 {% if enable_mail_notification %}
 <div class="modal fade" id="emailModal" tabindex="-1" role="dialog">
   <div class="modal-dialog modal-xl" role="document">