diff --git a/lookyloo/__init__.py b/lookyloo/__init__.py index 5423f649..9930ba72 100644 --- a/lookyloo/__init__.py +++ b/lookyloo/__init__.py @@ -3,9 +3,8 @@ import json -from har2tree import CrawledTree, hostname_treestyle +from har2tree import CrawledTree from scrapysplashwrapper import crawl -from .ete3_webserver import NodeActions, WebTreeHandler from flask import Flask, render_template, request from flask_bootstrap import Bootstrap @@ -29,10 +28,7 @@ def load_tree(report_dir): ct = CrawledTree(har_files) ct.find_parents() ct.join_trees() - ct.root_hartree.make_hostname_tree() - actions = NodeActions() - style = hostname_treestyle() - return WebTreeHandler(ct.root_hartree.hostname_tree, actions, style) + return ct.jsonify() @app.route('/scrape', methods=['GET', 'POST']) @@ -70,9 +66,8 @@ def get_report_dirs(): @app.route('/tree/', methods=['GET']) def tree(tree_id): report_dir = get_report_dirs()[tree_id] - tree = load_tree(report_dir) - nodes, faces, base64 = tree.redraw() - return render_template('tree.html', nodes=nodes, faces=faces, base64_img=base64) + tree_json = load_tree(report_dir) + return render_template('tree.html', tree_json=tree_json) @app.route('/', methods=['GET']) diff --git a/lookyloo/static/cookie_in_url.png b/lookyloo/static/cookie_in_url.png new file mode 100644 index 00000000..03959767 Binary files /dev/null and b/lookyloo/static/cookie_in_url.png differ diff --git a/lookyloo/static/cookie_read.png b/lookyloo/static/cookie_read.png new file mode 100644 index 00000000..0e18a014 Binary files /dev/null and b/lookyloo/static/cookie_read.png differ diff --git a/lookyloo/static/cookie_received.png b/lookyloo/static/cookie_received.png new file mode 100644 index 00000000..286432ae Binary files /dev/null and b/lookyloo/static/cookie_received.png differ diff --git a/lookyloo/static/ete.css b/lookyloo/static/ete.css deleted file mode 100644 index dbba2447..00000000 --- a/lookyloo/static/ete.css +++ /dev/null @@ -1,8 +0,0 @@ -#highlighter { - position: absolute; - visibility: visible; - z-index:100; - top:0; left:0; - width: 70px; height: 70px; - border: 2px solid indianred; -} diff --git a/lookyloo/static/ete.js b/lookyloo/static/ete.js deleted file mode 100644 index 5525187c..00000000 --- a/lookyloo/static/ete.js +++ /dev/null @@ -1,19 +0,0 @@ -function highlight_node(x, y, width, height){ - //console.log(treeid, nodeid, x, y, width, height); - var img = $('#img'); - var offset = img.offset(); - // console.log(img); - // console.log(offset); - - $("#highlighter").show(); - $("#highlighter").css("top", offset.top+y-1); - $("#highlighter").css("left", offset.left+x-1); - $("#highlighter").css("width", width+1); - $("#highlighter").css("height", height+1); - -} -function unhighlight_node(){ - // console.log("unhighlight"); - $("#highlighter").hide(); -} - diff --git a/lookyloo/static/javascript.png b/lookyloo/static/javascript.png new file mode 100644 index 00000000..e7a85697 Binary files /dev/null and b/lookyloo/static/javascript.png differ diff --git a/lookyloo/static/redirect.png b/lookyloo/static/redirect.png new file mode 100644 index 00000000..6bbf0e4f Binary files /dev/null and b/lookyloo/static/redirect.png differ diff --git a/lookyloo/static/tree.css b/lookyloo/static/tree.css new file mode 100644 index 00000000..fa4be6fc --- /dev/null +++ b/lookyloo/static/tree.css @@ -0,0 +1,15 @@ +.node circle { + fill: #fff; + stroke: steelblue; + stroke-width: 3px; +} + +.node text { + font: 12px sans-serif; +} + +.link { + fill: none; + stroke: #ccc; + stroke-width: 2px; +} diff --git a/lookyloo/static/tree.js b/lookyloo/static/tree.js new file mode 100644 index 00000000..8d7c3f3e --- /dev/null +++ b/lookyloo/static/tree.js @@ -0,0 +1,306 @@ +// From : https://bl.ocks.org/d3noob/43a860bc0024792f8803bba8ca0d5ecd + +// Set the dimensions and margins of the diagram +var margin = {top: 20, right: 200, bottom: 30, left: 90}, + width = 9600 - margin.left - margin.right, + height = 10000 - margin.top - margin.bottom; + +// append the svg object to the body of the page +// appends a 'group' element to 'svg' +// moves the 'group' element to the top left margin +var svg = d3.select("body").append("svg") + .attr("width", width + margin.right + margin.left) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + + margin.left + "," + margin.top + ")"); + +var i = 0, + duration = 750, + root; + +// declares a tree layout and assigns the size +var treemap = d3.tree().size([height, width]); + +// Assigns parent, children, height, depth +root = d3.hierarchy(treeData, function(d) { return d.children; }); +root.x0 = height / 2; +root.y0 = 0; + +// Collapse after the second level +// root.children.forEach(collapse); + +update(root); + + +// Collapse the node and all it's children +function collapse(d) { + if(d.children) { + d._children = d.children + d._children.forEach(collapse) + d.children = null + } +} + +function getBB(selection) { + selection.each(function(d) { + d.data.total_width = d.data.total_width ? d.data.total_width : 0; + d.data.total_width += this.getBBox().width; + }) +} + +function update(source) { + + // Assigns the x and y position for the nodes + var treeData = treemap(root); + + // Compute the new tree layout. + var nodes = treeData.descendants(), + links = treeData.descendants().slice(1); + + // Normalize for fixed-depth. + nodes.forEach(function(d){ d.y = d.depth * 180}); + + // ****************** Nodes section *************************** + + // Update the nodes... + // TODO: set that ID to the ete3 node ID + var node = svg.selectAll('g.node') + .data(nodes, function(d) {return d.id || (d.id = ++i); }); + + // Enter any new modes at the parent's previous position. + var nodeEnter = node.enter().append('g') + .attr('class', 'node') + .attr("transform", function(d) { + return "translate(" + source.y0 + "," + source.x0 + ")"; + }) + .on('click', click); + + // Add Circle for the nodes + nodeEnter.append('circle') + .attr('class', 'node') + .attr('r', 1e-6) + .style("fill", function(d) { + return d._children ? "lightsteelblue" : "#fff"; + }); + + // Avoid hiding the content after the circle + var nodeContent = nodeEnter + .append('svg') + .attr('x', 10) + .attr('y', -20) + .attr('width', function(d) { return d.data.name.length + 'em'; }); + + // Add labels for the nodes + nodeContent.append("text") + .attr('dy', '.9em') + .attr("stroke", "white") + .style("font-size", "16px") + .attr("stroke-width", ".2px") + .style("opacity", .9) + .attr('width', function(d) { return d.data.name.length + 'em'; }) + .text(function(d) { return d.data.name; }); + + // Put all the icone in one sub svg document + var icons = nodeEnter + .append('svg') + .attr('x', 10) + .attr('y', 10); + + // Add JavaScript information + var jsContent = icons + .append('svg'); + + jsContent.filter(function(d,i){ + return d.data.js > 0; + }).append('image') + .attr("width", 16) + .attr("height", 16) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr("xlink:href", "/static/javascript.png").call(getBB); + + jsContent.filter(function(d,i){ + return d.data.js > 0; + }).append('text') + .attr("dy", 8) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr('width', function(d) { return d.data.js.toString().length + 'em'; }) + .text(function(d) { return d.data.js; }).call(getBB); + + + // Add Cookie read information + var cookieReadContent = icons + .append('svg'); + + cookieReadContent.filter(function(d,i){ + return d.data.request_cookie > 0; + }).append('image') + .attr("width", 16) + .attr("height", 16) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr("xlink:href", "/static/cookie_read.png").call(getBB); + + cookieReadContent.filter(function(d,i){ + return d.data.request_cookie > 0; + }).append('text') + .attr("dy", 8) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr('width', function(d) { return d.data.request_cookie.toString().length + 'em'; }) + .text(function(d) { return d.data.request_cookie; }).call(getBB); + + // Add Cookie set information + var cookieSetContent = icons + .append('svg'); + + cookieSetContent.filter(function(d,i){ + return d.data.response_cookie > 0; + }).append('image') + .attr("width", 16) + .attr("height", 16) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr("xlink:href", "/static/cookie_received.png").call(getBB); + + cookieSetContent.filter(function(d,i){ + return d.data.response_cookie > 0; + }).append('text') + .attr("dy", 8) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr('width', function(d) { return d.data.response_cookie.toString().length + 'em'; }) + .text(function(d) { return d.data.response_cookie; }).call(getBB); + + // Add redirect information + var redirectContent = icons + .append('svg'); + + redirectContent.filter(function(d,i){ + return d.data.redirect > 0; + }).append('image') + .attr("width", 16) + .attr("height", 16) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width +1 : 0 }) + .attr("xlink:href", "/static/redirect.png").call(getBB); + + redirectContent.filter(function(d,i){ + return d.data.redirect > 0; + }).append('text') + .attr("dy", 8) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 2 : 0 }) + .attr('width', function(d) { return d.data.redirect.toString().length + 'em'; }) + .text(function(d) { return d.data.redirect; }).call(getBB); + + // Add cookie in URL information + var cookieURLContent = icons + .append('svg'); + + cookieURLContent.filter(function(d,i){ + return d.data.redirect_to_nothing > 0; + }).append('image') + .attr("width", 16) + .attr("height", 16) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .attr("xlink:href", "/static/cookie_in_url.png").call(getBB); + + cookieURLContent.filter(function(d,i){ + return d.data.redirect_to_nothing > 0; + }).append('text') + .attr("dy", 8) + .attr('x', function(d) { return d.data.total_width ? d.data.total_width + 1 : 0 }) + .text(function(d) { return d.data.redirect_to_nohing; }).call(getBB); + + + // UPDATE + var nodeUpdate = nodeEnter.merge(node); + + // Transition to the proper position for the node + nodeUpdate.transition() + .duration(duration) + .attr("transform", function(d) { + return "translate(" + d.y + "," + d.x + ")"; + }); + + // Update the node attributes and style + nodeUpdate.select('circle.node') + .attr('r', 10) + .style("fill", function(d) { + return d._children ? "lightsteelblue" : "#fff"; + }) + .attr('cursor', 'pointer'); + + + // Remove any exiting nodes + var nodeExit = node.exit().transition() + .duration(duration) + .attr("transform", function(d) { + return "translate(" + source.y + "," + source.x + ")"; + }) + .remove(); + + // On exit reduce the node circles size to 0 + nodeExit.select('circle') + .attr('r', 1e-6); + + // On exit reduce the opacity of text labels + nodeExit.select('text') + .style('fill-opacity', 1e-6); + + // ****************** links section *************************** + + // Update the links... + var link = svg.selectAll('path.link') + .data(links, function(d) { return d.id; }); + + // Enter any new links at the parent's previous position. + var linkEnter = link.enter().insert('path', "g") + .attr("class", "link") + .attr('d', function(d){ + var o = {x: source.x0, y: source.y0} + return diagonal(o, o) + }); + + // UPDATE + var linkUpdate = linkEnter.merge(link); + + // Transition back to the parent element position + linkUpdate.transition() + .duration(duration) + .attr('d', function(d){ return diagonal(d, d.parent) }); + + // Remove any exiting links + var linkExit = link.exit().transition() + .duration(duration) + .attr('d', function(d) { + var o = {x: source.x, y: source.y} + return diagonal(o, o) + }) + .remove(); + + // Store the old positions for transition. + nodes.forEach(function(d){ + d.x0 = d.x; + d.y0 = d.y; + }); + + // Creates a curved (diagonal) path from parent to the child nodes + function diagonal(s, d) { + + path = `M ${s.y} ${s.x} + C ${(s.y + d.y) / 2} ${s.x}, + ${(s.y + d.y) / 2} ${d.x}, + ${d.y} ${d.x}` + + return path + } + + // Toggle children on click. + function click(d) { + if (d.children) { + d._children = d.children; + d.children = null; + } else { + d.children = d._children; + d._children = null; + } + update(d); + } +} + diff --git a/lookyloo/templates/main.html b/lookyloo/templates/main.html index 0a2674d4..1cb48e50 100644 --- a/lookyloo/templates/main.html +++ b/lookyloo/templates/main.html @@ -2,10 +2,11 @@ {% block scripts %} {{ super() }} - + + {% endblock %} {% block head %} {{ super() }} - + {% endblock %} diff --git a/lookyloo/templates/tree.html b/lookyloo/templates/tree.html index 00a87de2..ea5d112a 100644 --- a/lookyloo/templates/tree.html +++ b/lookyloo/templates/tree.html @@ -2,30 +2,10 @@ {% block title %}Tree{% endblock %} + + {% block content %} - - - {% for x1, y1, x2, y2, nodeid, text, area in nodes %} - - {% endfor %} - {% for x1, y1, x2, y2, nodeid, text, area in faces %} - - {% endfor %} - -
- -
Powered by etetoolkit
-
+ {% endblock %}