Merge branch 'Felalex57-main'

lookyloo/lookyloo.py

@@ -829,7 +829,7 @@ class Lookyloo():
 
     def _capture(self, url: str, *, cookies_pseudofile: Optional[Union[BufferedIOBase, str]]=None,
                  depth: int=1, listing: bool=True, user_agent: Optional[str]=None,
-                 referer: str='', perma_uuid: Optional[str]=None, os: Optional[str]=None,
+                 referer: str='', proxy: str='', perma_uuid: Optional[str]=None, os: Optional[str]=None,
                  browser: Optional[str]=None, parent: Optional[str]=None) -> Union[bool, str]:
         '''Launch a capture'''
         url = url.strip()
@@ -866,7 +866,7 @@ class Lookyloo():
         self.logger.info(f'Capturing {url}')
         try:
             items = crawl(self.splash_url, url, cookies=cookies, depth=depth, user_agent=ua,
-                          referer=referer, log_enabled=True, log_level=get_config('generic', 'splash_loglevel'))
+                          referer=referer, proxy=proxy, log_enabled=True, log_level=get_config('generic', 'splash_loglevel'))
         except Exception as e:
             self.logger.critical(f'Something went terribly wrong when capturing {url}.')
             raise e

poetry.lock

@@ -813,11 +813,11 @@ python-versions = "*"
 
 [[package]]
 name = "pyrsistent"
-version = "0.17.3"
+version = "0.18.0"
 description = "Persistent/Functional/Immutable data structures"
 category = "main"
 optional = false
-python-versions = ">=3.5"
+python-versions = ">=3.6"
 
 [[package]]
 name = "pysanejs"
@@ -937,7 +937,7 @@ python-versions = "*"
 
 [[package]]
 name = "scrapysplashwrapper"
-version = "1.6.1"
+version = "1.6.2"
 description = "Scrapy splash wrapper as a standalone library."
 category = "main"
 optional = false
@@ -1215,7 +1215,7 @@ misp = ["python-magic", "pydeep"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "2b6a56502c3b03d210d88cc20ba33bfc12e25e8ae920f205a9c836313924c7ec"
+content-hash = "6651067bb5998433d83d7b2829a031fd1499a8304e39cbbedd9f9fb514e73c89"
 
 [metadata.files]
 aiohttp = [
@@ -1830,7 +1830,27 @@ pypydispatcher = [
     {file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"},
 ]
 pyrsistent = [
-    {file = "pyrsistent-0.17.3.tar.gz", hash = "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e"},
+    {file = "pyrsistent-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f4c8cabb46ff8e5d61f56a037974228e978f26bfefce4f61a4b1ac0ba7a2ab72"},
+    {file = "pyrsistent-0.18.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:da6e5e818d18459fa46fac0a4a4e543507fe1110e808101277c5a2b5bab0cd2d"},
+    {file = "pyrsistent-0.18.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5e4395bbf841693eaebaa5bb5c8f5cdbb1d139e07c975c682ec4e4f8126e03d2"},
+    {file = "pyrsistent-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:527be2bfa8dc80f6f8ddd65242ba476a6c4fb4e3aedbf281dfbac1b1ed4165b1"},
+    {file = "pyrsistent-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2aaf19dc8ce517a8653746d98e962ef480ff34b6bc563fc067be6401ffb457c7"},
+    {file = "pyrsistent-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58a70d93fb79dc585b21f9d72487b929a6fe58da0754fa4cb9f279bb92369396"},
+    {file = "pyrsistent-0.18.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4916c10896721e472ee12c95cdc2891ce5890898d2f9907b1b4ae0f53588b710"},
+    {file = "pyrsistent-0.18.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:73ff61b1411e3fb0ba144b8f08d6749749775fe89688093e1efef9839d2dcc35"},
+    {file = "pyrsistent-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:b29b869cf58412ca5738d23691e96d8aff535e17390128a1a52717c9a109da4f"},
+    {file = "pyrsistent-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:097b96f129dd36a8c9e33594e7ebb151b1515eb52cceb08474c10a5479e799f2"},
+    {file = "pyrsistent-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:772e94c2c6864f2cd2ffbe58bb3bdefbe2a32afa0acb1a77e472aac831f83427"},
+    {file = "pyrsistent-0.18.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c1a9ff320fa699337e05edcaae79ef8c2880b52720bc031b219e5b5008ebbdef"},
+    {file = "pyrsistent-0.18.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cd3caef37a415fd0dae6148a1b6957a8c5f275a62cca02e18474608cb263640c"},
+    {file = "pyrsistent-0.18.0-cp38-cp38-win32.whl", hash = "sha256:e79d94ca58fcafef6395f6352383fa1a76922268fa02caa2272fff501c2fdc78"},
+    {file = "pyrsistent-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:a0c772d791c38bbc77be659af29bb14c38ced151433592e326361610250c605b"},
+    {file = "pyrsistent-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d5ec194c9c573aafaceebf05fc400656722793dac57f254cd4741f3c27ae57b4"},
+    {file = "pyrsistent-0.18.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:6b5eed00e597b5b5773b4ca30bd48a5774ef1e96f2a45d105db5b4ebb4bca680"},
+    {file = "pyrsistent-0.18.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:48578680353f41dca1ca3dc48629fb77dfc745128b56fc01096b2530c13fd426"},
+    {file = "pyrsistent-0.18.0-cp39-cp39-win32.whl", hash = "sha256:f3ef98d7b76da5eb19c37fda834d50262ff9167c65658d1d8f974d2e4d90676b"},
+    {file = "pyrsistent-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:404e1f1d254d314d55adb8d87f4f465c8693d6f902f67eb6ef5b4526dc58e6ea"},
+    {file = "pyrsistent-0.18.0.tar.gz", hash = "sha256:773c781216f8c2900b42a7b638d5b517bb134ae1acbebe4d1e8f1f41ea60eb4b"},
 ]
 pysanejs = [
     {file = "pysanejs-1.4-py3-none-any.whl", hash = "sha256:55d1c14a4e0a8658cefb622b175c62901e33bea24fbc68f2aa420f592ed687e8"},
@@ -1869,8 +1889,8 @@ scrapy-splash = [
     {file = "scrapy_splash-0.7.2-py2.py3-none-any.whl", hash = "sha256:71ac958370f8732fec746a25a8235b03a4d3c4c93a59be51aa8e910a08cfe511"},
 ]
 scrapysplashwrapper = [
-    {file = "scrapysplashwrapper-1.6.1-py3-none-any.whl", hash = "sha256:9e0535a48095e7762c4cc54226e00e41d6a483e64dc6908988fb1622704f5207"},
-    {file = "scrapysplashwrapper-1.6.1.tar.gz", hash = "sha256:46bfd2c30ccd387662eef1ee47187f41d5f6f9d1259f89db88eacaad358fec47"},
+    {file = "scrapysplashwrapper-1.6.2-py3-none-any.whl", hash = "sha256:c9c190052c14d978f25138210a9ad227d563636a2c7e6bd8bcca5b43b38e5228"},
+    {file = "scrapysplashwrapper-1.6.2.tar.gz", hash = "sha256:eb80cfdf87823b9e5507e097f0e0b02bc0a9786bbdedce42be9d0339d8d84032"},
 ]
 service-identity = [
     {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"},

pyproject.toml

@@ -45,7 +45,7 @@ bootstrap-flask = "^1.7.0"
 defang = "^0.5.3"
 vt-py = "^0.6.2"
 pyeupi = "^1.1"
-scrapysplashwrapper = "^1.6.0"
+scrapysplashwrapper = "^1.6.2"
 pysanejs = "^1.4"
 har2tree = "^1.6.1"
 pylookyloo = "^1.6"

website/web/__init__.py

@@ -12,7 +12,7 @@ import calendar
 from typing import Optional, Dict, Any, Union, List
 import logging
 import hashlib
-from urllib.parse import quote_plus, unquote_plus
+from urllib.parse import quote_plus, unquote_plus, urlparse
 import time
 
 from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify
@@ -758,6 +758,19 @@ def capture_web():
         if request.form.get('referer'):
             capture_query['referer'] = request.form['referer']
 
+        if request.form.get('proxy'):
+            parsed_proxy = urlparse(request.form['proxy'])
+            if parsed_proxy.scheme and parsed_proxy.hostname and parsed_proxy.port:
+                if parsed_proxy.scheme in ['http', 'https', 'socks5']:
+                    if (parsed_proxy.username and parsed_proxy.password) != (not parsed_proxy.username and not parsed_proxy.password):
+                        capture_query['proxy'] = request.form['proxy']
+                    else:
+                        flash('You need to enter a username AND a password for your proxy.', 'error')
+                else:
+                    flash('Proxy scheme not supported: must be http(s) or socks5.', 'error')
+            else:
+                flash('Invalid proxy: Check that you entered a scheme, a hostname and a port.', 'error')
+
         perma_uuid = lookyloo.enqueue_capture(capture_query, source='web', user=user, authenticated=flask_login.current_user.is_authenticated)
         time.sleep(30)
         return redirect(url_for('tree', tree_uuid=perma_uuid))

website/web/templates/capture.html

@@ -67,6 +67,13 @@
         </div>
       </div>
 
+      <div class="form-group row">
+        <label for="proxy" class="col-sm-2 col-form-label">Proxy (Optional):</label>
+        <div class="col-sm-10">
+          <input type="text" class="form-control" name="proxy" id="proxy" placeholder="Expected format: [scheme]://[username]:[password]@[hostname]:[port]">
+        </div>
+      </div>
+
       <div class="form-group row">
         <label for="cookies" class="col-sm-6">Cookies (JSON export from the Firefox plugin Cookie Quick Manager)</label>
         <div class="col-sm-4">