diff --git a/Pipfile b/Pipfile index 0fe740f0..00a9ed05 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ verify_ssl = true [packages] scrapysplashwrapper = {editable = true,git = "https://github.com/viper-framework/ScrapySplashWrapper.git"} har2tree = {editable = true,git = "https://github.com/viper-framework/har2tree.git"} +pysanejs = {editable = true,git = "https://github.com/CIRCL/PySaneJS.git"} requests = "*" flask = "*" flask-bootstrap = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5e387482..7c35c8a3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6b8daa132362733ad56e82a23ffc6141ebbb39c5ed74054b1d445aeec88d5b30" + "sha256": "6468b6eae67fd44ea812c2d4d5cdfe2faf39a109374cc8ef526b6c9419927968" }, "pipfile-spec": 6, "requires": { @@ -378,10 +378,10 @@ }, "pyasn1-modules": { "hashes": [ - "sha256:642afdabb681d39f5948fd5477764d94faf17ce40e5691e9998b52815fbb4e71", - "sha256:d14fcb29dabecba3d7b360bf72327c26c385248a5d603cf6be5f566ce999b261" + "sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b", + "sha256:a52090e8c5841ebbf08ae455146792d9ef3e8445b21055d3a3b7ed9c712b7c7c" ], - "version": "==0.2.3" + "version": "==0.2.4" }, "pycparser": { "hashes": [ @@ -410,6 +410,11 @@ ], "version": "==19.0.0" }, + "pysanejs": { + "editable": true, + "git": "https://github.com/CIRCL/PySaneJS.git", + "ref": "a91ebf014754b1b84e1c2874759315446d4a6b85" + }, "queuelib": { "hashes": [ "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17", diff --git a/lookyloo/__init__.py b/lookyloo/__init__.py index bfddc17c..8a4c1839 100644 --- a/lookyloo/__init__.py +++ b/lookyloo/__init__.py @@ -20,10 +20,9 @@ from zipfile import ZipFile, ZIP_DEFLATED from io import BytesIO import base64 import socket -from urllib.parse import urlparse import os -import requests +from pysanejs import SaneJS from .helpers import get_homedir @@ -63,8 +62,8 @@ def is_open(ip, port): if SANE_JS: - parsed = urlparse(SANE_JS) - if is_open(parsed.hostname, parsed.port): + sanejs = SaneJS(SANE_JS) + if sanejs.is_up: has_sane_js = True else: has_sane_js = False @@ -94,11 +93,10 @@ def load_tree(report_dir): return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url -def sane_js_query(sha512, details=False): +def sane_js_query(sha512): if has_sane_js: - r = requests.post(SANE_JS, json={"sha512": sha512, 'details': details}) - return r.json() - return {'exists': False} + return sanejs.sha512(sha512) + return {'response': []} @app.route('/scrape', methods=['GET', 'POST']) @@ -168,9 +166,9 @@ def hostnode_details(node_uuid): urls = [] for url in hostnode.urls: if hasattr(url, 'body_hash'): - sane_js_r = sane_js_query(url.body_hash, details=True) - if sane_js_r['exists']: - url.add_feature('sane_js_details', sane_js_r['details']) + sane_js_r = sane_js_query(url.body_hash) + if sane_js_r.get('response'): + url.add_feature('sane_js_details', sane_js_r['response']) print(url.sane_js_details) urls.append(url.to_json()) return json.dumps(urls) diff --git a/lookyloo/static/tree.js b/lookyloo/static/tree.js index cf3626cd..50b12b29 100644 --- a/lookyloo/static/tree.js +++ b/lookyloo/static/tree.js @@ -313,7 +313,7 @@ function icon_list(parent_svg, relative_x_pos, relative_y_pos) { icons.filter(function(d){ if (d.data.sane_js_details) { - d.libname = d.data.sane_js_details[0]['libname']; + d.libinfo = d.data.sane_js_details[0]; return d.data.sane_js_details; } return false; @@ -321,7 +321,7 @@ function icon_list(parent_svg, relative_x_pos, relative_y_pos) { .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 5 : 0 }) .attr('y', 15) .style("font-size", "15px") - .text(function(d) { return 'Library name: ' + d.libname }).call(getBB); + .text(function(d) { return 'Library inforamtion: ' + d.libinfo }).call(getBB); } function text_entry(parent_svg, relative_x_pos, relative_y_pos, onclick_callback) {