new: Cookies investigation

lookyloo/lookyloo.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 
 import base64
+from collections import defaultdict
+
 from datetime import datetime
 from email.message import EmailMessage
 from io import BufferedIOBase, BytesIO
@@ -12,7 +14,7 @@ from pathlib import Path
 import pickle
 import smtplib
 import socket
-from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping
+from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set
 from urllib.parse import urlsplit
 from uuid import uuid4
 from zipfile import ZipFile
@@ -423,3 +425,61 @@ class Lookyloo():
 
         self._set_capture_cache(dirpath)
         return perma_uuid
+
+    def get_hostnode_investigator(self, capture_dir: Path, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]:
+        ct = self._load_pickle(capture_dir / 'tree.pickle')
+        if not ct:
+            raise MissingUUID(f'Unable to find {capture_dir}')
+        hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
+        if not hostnode:
+            raise MissingUUID(f'Unable to find UUID {node_uuid} in {capture_dir}')
+
+        sanejs_lookups: Dict[str, List[str]] = {}
+        if hasattr(self, 'sanejs') and self.sanejs.available:
+            to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
+            sanejs_lookups = self.sanejs.hashes_lookup(to_lookup)
+
+        urls: List[Dict[str, Any]] = []
+        for url in hostnode.urls:
+            # For the popup, we need:
+            # * https vs http
+            # * everything after the domain
+            # * the full URL
+            to_append: Dict[str, Any] = {
+                'encrypted': url.name.startswith('https'),
+                'url_path': url.name.split('/', 3)[-1],
+                'url_object': url
+            }
+
+            # Optional: SaneJS information
+            if hasattr(url, 'body_hash') and url.body_hash in sanejs_lookups:
+                if sanejs_lookups[url.body_hash]:
+                    if isinstance(sanejs_lookups[url.body_hash], list):
+                        libname, version, path = sanejs_lookups[url.body_hash][0].split("|")
+                        other_files = len(sanejs_lookups[url.body_hash])
+                        to_append['sane_js'] = (libname, version, path, other_files)
+                    else:
+                        # Predefined generic file
+                        to_append['sane_js'] = sanejs_lookups[url.body_hash]
+
+            # Optional: Cookies sent to server in request -> map to nodes who set the cookie in response
+            if hasattr(url, 'cookies_sent'):
+                to_display: Dict[str, Set[Tuple[str, str]]] = defaultdict(set)
+                for cookie, contexts in url.cookies_sent.items():
+                    if not contexts:
+                        # FIXME Locally created?
+                        continue
+                    for context in contexts:
+                        to_display[cookie].add((context['setter'].hostname, context['setter'].hostnode_uuid))
+                to_append['cookies_sent'] = to_display
+
+            # Optional: Cookies received from server in response -> map to nodes who send the cookie in request
+            if hasattr(url, 'cookies_received'):
+                to_display = defaultdict(set)
+                for domain, c_received, is_3rd_party in url.cookies_received:
+                    for url_node in ct.root_hartree.cookies_sent[c_received]:
+                        to_display[c_received].add((url_node.hostname, url_node.hostnode_uuid))
+                to_append['cookies_received'] = to_display
+
+            urls.append(to_append)
+        return hostnode, urls

lookyloo/modules.py

@@ -49,6 +49,7 @@ class SaneJavaScript():
         "16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516": "This is a 1*1 pixel GIF",
         "01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb": "This is a 1*1 pixel GIF",
         "71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746": "This is a 1*1 pixel GIF",
+        "49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a": "This is a 1*1 pixel GIF",
         # "": "This is a 1*1 pixel GIF",
         "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG",
         "dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99": "This is a 1*1 pixel PNG",

poetry.lock

@@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221"
 six = "^1.14.0"
 
 [package.source]
-reference = "888dab5e75b7da56a84c74e7e55c4cf1c79cbf0b"
+reference = "6ee33972962351214413e07c33559691f1ae9b9d"
 type = "git"
 url = "https://github.com/viper-framework/har2tree.git"
 [[package]]
@@ -739,7 +739,7 @@ description = "pytest: simple powerful testing with Python"
 name = "pytest"
 optional = false
 python-versions = ">=3.5"
-version = "5.4.2"
+version = "5.4.3"
 
 [package.dependencies]
 atomicwrites = ">=1.0"
@@ -788,7 +788,7 @@ description = "Python client for Redis key-value store"
 name = "redis"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-version = "3.5.2"
+version = "3.5.3"
 
 [package.extras]
 hiredis = ["hiredis (>=0.1.3)"]
@@ -1011,11 +1011,11 @@ six = ">=1.4.1"
 
 [[package]]
 category = "main"
-description = "Measures number of Terminal column cells of wide-character codes"
+description = "Measures the displayed width of unicode strings in a terminal"
 name = "wcwidth"
 optional = false
 python-versions = "*"
-version = "0.1.9"
+version = "0.2.3"
 
 [[package]]
 category = "main"
@@ -1502,8 +1502,8 @@ pypydispatcher = [
 ]
 pysanejs = []
 pytest = [
-    {file = "pytest-5.4.2-py3-none-any.whl", hash = "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3"},
+    {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
-    {file = "pytest-5.4.2.tar.gz", hash = "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698"},
+    {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
 ]
 pytest-httpserver = [
     {file = "pytest_httpserver-0.3.4-py3-none-any.whl", hash = "sha256:7feab352b2626d1a0ecdebffcac5e5875979f08ad7e621b2289980ce8f6ebc5b"},
@@ -1514,8 +1514,8 @@ queuelib = [
     {file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"},
 ]
 redis = [
-    {file = "redis-3.5.2-py2.py3-none-any.whl", hash = "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242"},
+    {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"},
-    {file = "redis-3.5.2.tar.gz", hash = "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251"},
+    {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"},
 ]
 requests = [
     {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"},
@@ -1615,8 +1615,8 @@ w3lib = [
     {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"},
 ]
 wcwidth = [
-    {file = "wcwidth-0.1.9-py2.py3-none-any.whl", hash = "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1"},
+    {file = "wcwidth-0.2.3-py2.py3-none-any.whl", hash = "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6"},
-    {file = "wcwidth-0.1.9.tar.gz", hash = "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"},
+    {file = "wcwidth-0.2.3.tar.gz", hash = "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830"},
 ]
 werkzeug = [
     {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},

website/web/__init__.py

@@ -17,7 +17,7 @@ from lookyloo.lookyloo import Lookyloo
 from lookyloo.exceptions import NoValidHarFile
 from .proxied import ReverseProxied
 
-from typing import Optional, Dict, Any, List, Tuple
+from typing import Optional, Dict, Any
 
 import logging
 
@@ -144,7 +144,7 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
     capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
     if not capture_dir:
         return
-    hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
+
     keys_response = {
         'js': "/static/javascript.png",
         'exe': "/static/exe.png",
@@ -164,29 +164,8 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
         'request_cookie': "/static/cookie_read.png",
     }
 
-    sanejs_lookups: Dict[str, List[str]] = {}
+    hostnode, urls = lookyloo.get_hostnode_investigator(capture_dir, node_uuid)
-    if hasattr(lookyloo, 'sanejs') and lookyloo.sanejs.available:
-        to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
-        sanejs_lookups = lookyloo.sanejs.hashes_lookup(to_lookup)
 
-    urls: List[Tuple[bool, str, str]] = []
-    for url in hostnode.urls:
-        if hasattr(url, 'body_hash') and url.body_hash in sanejs_lookups:
-            url.add_feature('sane_js_details', sanejs_lookups[url.body_hash])
-            if sanejs_lookups[url.body_hash]:
-                if isinstance(sanejs_lookups[url.body_hash], list):
-                    libname, version, path = sanejs_lookups[url.body_hash][0].split("|")
-                    other_files = len(sanejs_lookups[url.body_hash])
-                    url.add_feature('sane_js_details_to_print', (libname, version, path, other_files))
-                else:
-                    # Predefined generic file
-                    url.add_feature('sane_js_details_to_print', sanejs_lookups[url.body_hash])
-
-        # For the popup, we need:
-        # * https vs http
-        # * everything after the domain
-        # * the full URL
-        urls.append((url.name.startswith('https'), url.name.split('/', 3)[-1], url))
     return render_template('hostname_popup.html',
                            tree_uuid=tree_uuid,
                            hostname_uuid=node_uuid,

website/web/templates/hostname_popup.html

@@ -5,8 +5,8 @@
 {% block scripts %}
   {{ super() }}
   <script>
-      function whereAmI() {
+      function whereAmI(hostname_uuid) {
-          window.opener.ProcessChildMessage("{{ hostname_uuid }}");
+          window.opener.ProcessChildMessage(hostname_uuid);
       };
   </script>
   <script>
@@ -44,22 +44,22 @@
 {% block content %}
   <center>
     <h3>{{ hostname }}</h3>
-    <button type="button" class="btn btn-info" onclick="whereAmI()">Locate node on tree</button>
+    <button type="button" class="btn btn-info" onclick="whereAmI('{{ hostname_uuid }}')">Locate node on tree</button>
     <a href="{{ url_for('hostnode_details_text', tree_uuid=tree_uuid, node_uuid=hostname_uuid) }}" class="btn btn-info" role="button">Get URLs as text</a>
   </center>
   <p>Click on the URL to get the content of the response</p>
   <ul class="list-group-flush">
-    {% for secure, path, url in urls %}
+    {% for url in urls %}
     <li class="list-group-item">
       <div class="h3">
         <button type="button" class="btn btn-default btn-copy js-copy"
-                data-toggle="tooltip" data-placement="bottom" data-copy="{{url.name}}" title="Copy to clipboard">
+                data-toggle="tooltip" data-placement="bottom" data-copy="{{ url['url_object'].name }}" title="Copy to clipboard">
           <svg class="bi bi-clipboard" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
             <path fill-rule="evenodd" d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
             <path fill-rule="evenodd" d="M9.5 1h-3a.5.5 0 0 0-.5.5v1a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-1a.5.5 0 0 0-.5-.5zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
           </svg>
         </button>
-        {% if secure %}
+        {% if url['encrypted'] %}
         <svg class="bi bi-lock" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
           <path fill-rule="evenodd" d="M11.5 8h-7a1 1 0 0 0-1 1v5a1 1 0 0 0 1 1h7a1 1 0 0 0 1-1V9a1 1 0 0 0-1-1zm-7-1a2 2 0 0 0-2 2v5a2 2 0 0 0 2 2h7a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2h-7zm0-3a3.5 3.5 0 1 1 7 0v3h-1V4a2.5 2.5 0 0 0-5 0v3h-1V4z"/>
         </svg>
@@ -68,7 +68,7 @@
           <path fill-rule="evenodd" d="M9.655 8H2.333c-.264 0-.398.068-.471.121a.73.73 0 0 0-.224.296 1.626 1.626 0 0 0-.138.59V14c0 .342.076.531.14.635.064.106.151.18.256.237a1.122 1.122 0 0 0 .436.127l.013.001h7.322c.264 0 .398-.068.471-.121a.73.73 0 0 0 .224-.296 1.627 1.627 0 0 0 .138-.59V9c0-.342-.076-.531-.14-.635a.658.658 0 0 0-.255-.237A1.122 1.122 0 0 0 9.655 8zm.012-1H2.333C.5 7 .5 9 .5 9v5c0 2 1.833 2 1.833 2h7.334c1.833 0 1.833-2 1.833-2V9c0-2-1.833-2-1.833-2zM8.5 4a3.5 3.5 0 1 1 7 0v3h-1V4a2.5 2.5 0 0 0-5 0v3h-1V4z"/>
         </svg>
         {%endif%}
-        ... /{{ path }}
+        ... /{{ url['url_path'] }}
       </div>
       <ul class="list-group">
         <li class="list-group-item">
@@ -76,9 +76,9 @@
 
           <div>
           {% for key, path in keys_response.items() %}
-            {% if url[key] %}
+            {% if url['url_object'][key] %}
               {% if key == "response_cookie" %}
-              <a href="{{ url_for('urlnode_response_cookies', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
+              <a href="{{ url_for('urlnode_response_cookies', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
                <img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
               </a>
               {% else %}
@@ -89,38 +89,55 @@
           </div>
 
           <div>
-          {% if not url.empty_response %}
+          {% if not url['url_object'].empty_response %}
-            <a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
+            <a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
               Download response body.
             </a></br>
-            Body size: {{ sizeof_fmt(url.body.getbuffer().nbytes) }}
+            Body size: {{ sizeof_fmt(url['url_object'].body.getbuffer().nbytes) }}
           {% else %}
             Empty body.
           {%endif%}
           </div>
 
           <div>
-            Status Code: {{ url.response['status'] }}
+            Status Code: {{ url['url_object'].response['status'] }}
           </div>
 
 
-          {% if url.sane_js_details_to_print %}
+          {% if url['sane_js'] %}
           <div>
-            {% if url.sane_js_details_to_print is string %}
+            {% if url['sane_js'] is string %}
-              {{ url.sane_js_details_to_print }}
+              {{ url['sane_js'] }}
             {% else %}
-              This file is known as part of <b>{{ url.sane_js_details_to_print[0] }}</b>
+              This file is known as part of <b>{{ url['sane_js'][0] }}</b>
-              version <b>{{ url.sane_js_details_to_print[1] }}</b>: <b>{{ url.sane_js_details_to_print[2] }}</b>.
+              version <b>{{ url['sane_js'][1] }}</b>: <b>{{ url['sane_js'][2] }}</b>.
-              {% if url.sane_js_details_to_print[3] > 1%}
+              {% if url['sane_js'][3] > 1%}
-              It is also present in <b>{{ url.sane_js_details_to_print[3] -1 }}</b> other libraries.
+              It is also present in <b>{{ url['sane_js'][3] -1 }}</b> other libraries.
               {%endif%}
             {%endif%}
           </div>
           {% endif %}
 
-          {% if url.set_third_party_cookies %}
+          {% if url['url_object'].set_third_party_cookies %}
           <div>This response contains 3rd party cookies.</div>
           {% endif %}
+          {% if url['cookies_received'] %}
+          <p class="h6">List of cookies received in the response to that URL</p>
+          <ul class="list-group">
+            {% for cookie, details in url['cookies_received'].items() %}
+            <li class="list-group-item">{{ cookie }}
+                <ul class="list-group">
+                {% for hostname, hostnode_uuid in details %}
+                  <li class="list-group-item"> {{ hostname }} -
+                  <button type="button" class="btn btn-info" onclick="whereAmI('{{ hostnode_uuid }}')">Show on tree node sending this cookie</button>
+                  </li>
+                {% endfor %}
+                </ul>
+            </li>
+            {% endfor %}
+          </ul>
+          </p>
+          {% endif %}
 
         </li>
 
@@ -128,9 +145,9 @@
           <p class="h4">Request</p>
           <div>
           {% for key, path in keys_request.items() %}
-            {% if url[key] %}
+            {% if url['url_object'][key] %}
               {% if key == "request_cookie" %}
-              <a href="{{ url_for('urlnode_request_cookies', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
+              <a href="{{ url_for('urlnode_request_cookies', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
                <img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
               </a>
               {% else %}
@@ -139,14 +156,30 @@
             {%endif%}
           {% endfor %}
           </div>
-          {% if url.posted_data %}
+          {% if url['url_object'].posted_data %}
-          <a href="{{ url_for('urlnode_post_request', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
+          <a href="{{ url_for('urlnode_post_request', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
               Download posted data
           </a></br>
-          Posted data size: {{ sizeof_fmt(url.posted_data|length) }}
+          Posted data size: {{ sizeof_fmt(url['url_object'].posted_data|length) }}
+          {% endif %}
+
+          {% if url['cookies_sent'] %}
+          <p class="h6">List of cookies sent in the request to that URL</p>
+          <ul class="list-group">
+            {% for cookie, details in url['cookies_sent'].items() %}
+            <li class="list-group-item">{{ cookie }}
+                <ul class="list-group">
+                {% for hostname, hostnode_uuid in details %}
+                  <li class="list-group-item"> {{ hostname }} -
+                  <button type="button" class="btn btn-info" onclick="whereAmI('{{ hostnode_uuid }}')">Show on tree node setting this cookie</button>
+                  </li>
+                {% endfor %}
+                </ul>
+            </li>
+            {% endfor %}
+          </ul>
+          </p>
           {% endif %}
-        </li>
-      </ul>
     </li>
     {% endfor %}
   </ul>