CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Use PySaneJS to query SaneJS

pull/27/head
Raphaël Vinot 2019-01-27 17:17:31 +01:00
parent 8059c342fc
commit 945511d6b3
4 changed files with 21 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -8,6 +8,7 @@ verify_ssl = true
[packages] [packages]
scrapysplashwrapper = {editable = true,git = "https://github.com/viper-framework/ScrapySplashWrapper.git"} scrapysplashwrapper = {editable = true,git = "https://github.com/viper-framework/ScrapySplashWrapper.git"}
har2tree = {editable = true,git = "https://github.com/viper-framework/har2tree.git"} har2tree = {editable = true,git = "https://github.com/viper-framework/har2tree.git"}
pysanejs = {editable = true,git = "https://github.com/CIRCL/PySaneJS.git"}
requests = "*" requests = "*"
flask = "*" flask = "*"
flask-bootstrap = "*" flask-bootstrap = "*"

13
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "6b8daa132362733ad56e82a23ffc6141ebbb39c5ed74054b1d445aeec88d5b30" "sha256": "6468b6eae67fd44ea812c2d4d5cdfe2faf39a109374cc8ef526b6c9419927968"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -378,10 +378,10 @@
}, },
"pyasn1-modules": { "pyasn1-modules": {
"hashes": [ "hashes": [
"sha256:642afdabb681d39f5948fd5477764d94faf17ce40e5691e9998b52815fbb4e71", "sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b",
"sha256:d14fcb29dabecba3d7b360bf72327c26c385248a5d603cf6be5f566ce999b261" "sha256:a52090e8c5841ebbf08ae455146792d9ef3e8445b21055d3a3b7ed9c712b7c7c"
], ],
"version": "==0.2.3" "version": "==0.2.4"
}, },
"pycparser": { "pycparser": {
"hashes": [ "hashes": [
@ -410,6 +410,11 @@
], ],
"version": "==19.0.0" "version": "==19.0.0"
}, },
"pysanejs": {
"editable": true,
"git": "https://github.com/CIRCL/PySaneJS.git",
"ref": "a91ebf014754b1b84e1c2874759315446d4a6b85"
},
"queuelib": { "queuelib": {
"hashes": [ "hashes": [
"sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17", "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17",

20
lookyloo/__init__.py
View File

@ -20,10 +20,9 @@ from zipfile import ZipFile, ZIP_DEFLATED
from io import BytesIO from io import BytesIO
import base64 import base64
import socket import socket
from urllib.parse import urlparse
import os import os
import requests from pysanejs import SaneJS
from .helpers import get_homedir from .helpers import get_homedir
@ -63,8 +62,8 @@ def is_open(ip, port):
if SANE_JS: if SANE_JS:
parsed = urlparse(SANE_JS) sanejs = SaneJS(SANE_JS)
if is_open(parsed.hostname, parsed.port): if sanejs.is_up:
has_sane_js = True has_sane_js = True
else: else:
has_sane_js = False has_sane_js = False
@ -94,11 +93,10 @@ def load_tree(report_dir):
return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url
def sane_js_query(sha512, details=False): def sane_js_query(sha512):
if has_sane_js: if has_sane_js:
r = requests.post(SANE_JS, json={"sha512": sha512, 'details': details}) return sanejs.sha512(sha512)
return r.json() return {'response': []}
return {'exists': False}
@app.route('/scrape', methods=['GET', 'POST']) @app.route('/scrape', methods=['GET', 'POST'])
@ -168,9 +166,9 @@ def hostnode_details(node_uuid):
urls = [] urls = []
for url in hostnode.urls: for url in hostnode.urls:
if hasattr(url, 'body_hash'): if hasattr(url, 'body_hash'):
sane_js_r = sane_js_query(url.body_hash, details=True) sane_js_r = sane_js_query(url.body_hash)
if sane_js_r['exists']: if sane_js_r.get('response'):
url.add_feature('sane_js_details', sane_js_r['details']) url.add_feature('sane_js_details', sane_js_r['response'])
print(url.sane_js_details) print(url.sane_js_details)
urls.append(url.to_json()) urls.append(url.to_json())
return json.dumps(urls) return json.dumps(urls)

4
lookyloo/static/tree.js
View File

@ -313,7 +313,7 @@ function icon_list(parent_svg, relative_x_pos, relative_y_pos) {
icons.filter(function(d){ icons.filter(function(d){
if (d.data.sane_js_details) { if (d.data.sane_js_details) {
d.libname = d.data.sane_js_details[0]['libname']; d.libinfo = d.data.sane_js_details[0];
return d.data.sane_js_details; return d.data.sane_js_details;
} }
return false; return false;
@ -321,7 +321,7 @@ function icon_list(parent_svg, relative_x_pos, relative_y_pos) {
.attr('x', function(d) { return d.data.total_width ? d.data.total_width + 5 : 0 }) .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 5 : 0 })
.attr('y', 15) .attr('y', 15)
.style("font-size", "15px") .style("font-size", "15px")
.text(function(d) { return 'Library name: ' + d.libname }).call(getBB); .text(function(d) { return 'Library inforamtion: ' + d.libinfo }).call(getBB);
} }
function text_entry(parent_svg, relative_x_pos, relative_y_pos, onclick_callback) { function text_entry(parent_svg, relative_x_pos, relative_y_pos, onclick_callback) {