From 2ae7d87420acda23cef99942ceaeab83fc04a5a2 Mon Sep 17 00:00:00 2001
From: "Fafner [_KeyZee_]" <FafnerKeyZee@users.noreply.github.com>
Date: Tue, 24 Nov 2020 16:44:43 +0100
Subject: [PATCH] Adding uniq_domains

---
 lookyloo/helpers.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py
index 053dad8d..56fd344b 100644
--- a/lookyloo/helpers.py
+++ b/lookyloo/helpers.py
@@ -29,6 +29,7 @@ except ImportError:
 
 from .exceptions import MissingEnv, CreateDirectoryException, ConfigError
 
+from urllib.parse import urlparse
 
 configs: Dict[str, Dict[str, Any]] = {}
 logger = logging.getLogger('Lookyloo - Helpers')
@@ -313,3 +314,10 @@ def remove_pickle_tree(capture_dir: Path) -> None:
     pickle_file = capture_dir / 'tree.pickle'
     if pickle_file.exists():
         pickle_file.unlink()
+
+def uniq_domains(uniq_urls):
+    domains = set()
+    for url in uniq_urls:
+        splitted = urlparse(url)
+        domains.add(splitted.hostname)
+    return domains