diff --git a/lookyloo/__init__.py b/lookyloo/__init__.py index 648872b8..090a714f 100644 --- a/lookyloo/__init__.py +++ b/lookyloo/__init__.py @@ -62,7 +62,7 @@ def scrape(): if request.form.get('url'): url = request.form.get('url') if not url.startswith('http'): - url = 'http://{}'.format(url) + url = f'http://{url}' depth = request.form.get('depth') if depth is None: depth = 1 @@ -98,6 +98,24 @@ def get_report_dirs(): return sorted(HAR_DIR.iterdir(), reverse=True) +@app.route('/tree/hostname//text', methods=['GET']) +def hostnode_details_text(node_uuid): + with open(session["tree"], 'rb') as f: + ct = pickle.load(f) + hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid) + urls = [] + for url in hostnode.urls: + urls.append(url.name) + content = '''# URLs + +{} +'''.format('\n'.join(urls)) + to_return = BytesIO(content.encode()) + to_return.seek(0) + return send_file(to_return, mimetype='text/markdown', + as_attachment=True, attachment_filename='file.md') + + @app.route('/tree/hostname/', methods=['GET']) def hostnode_details(node_uuid): with open(session["tree"], 'rb') as f: @@ -114,16 +132,18 @@ def urlnode_details(node_uuid): with open(session["tree"], 'rb') as f: ct = pickle.load(f) urlnode = ct.root_hartree.get_url_node_by_uuid(node_uuid) - to_return = BytesIO() + got_content = False if hasattr(urlnode, 'body'): - with ZipFile(to_return, 'a', ZIP_DEFLATED, False) as zfile: - zfile.writestr(urlnode.filename, urlnode.body.getvalue()) - to_return.seek(0) - # return send_file(urlnode.body, mimetype='application/zip', - # as_attachment=True, attachment_filename='file.zip') - with open('foo.bin', 'wb') as f: - f.write(to_return.getvalue()) + body_content = urlnode.body.getvalue() + if body_content: + got_content = True + with ZipFile(to_return, 'w', ZIP_DEFLATED) as zfile: + zfile.writestr(urlnode.filename, urlnode.body.getvalue()) + if not got_content: + with ZipFile(to_return, 'w', ZIP_DEFLATED) as zfile: + zfile.writestr('file.txt', b'Response body empty') + to_return.seek(0) return send_file(to_return, mimetype='application/zip', as_attachment=True, attachment_filename='file.zip') diff --git a/lookyloo/static/tree.js b/lookyloo/static/tree.js index df39dd32..f8427ae9 100644 --- a/lookyloo/static/tree.js +++ b/lookyloo/static/tree.js @@ -6,12 +6,19 @@ var margin = {top: 20, right: 200, bottom: 30, left: 90}, height = 10000 - margin.top - margin.bottom; var node_width = 0; +var max_overlay_width = 1500; var node_height = 45; var main_svg = d3.select("body").append("svg") .attr("width", width + margin.right + margin.left) .attr("height", height + margin.top + margin.bottom) +main_svg.append("clipPath") + .attr("id", "textOverlay") + .append("rect") + .attr('width', max_overlay_width - 25) + .attr('height', node_height); + // Add background pattern var pattern = main_svg.append("defs").append('pattern') .attr('id', 'backstripes') @@ -75,7 +82,7 @@ function getBB(selection) { }; function urlnode_click(d) { - var url = "url/" + d.data.uuid; + var url = "tree/url/" + d.data.uuid; d3.blob(url, {credentials: 'same-origin'}).then(function(data) { saveAs(data, 'file.zip'); }); @@ -102,7 +109,9 @@ function hostnode_click(d) { cur_node.append('line') .attr('id', 'overlay_link') - .style("stroke", "black"); + .style("opacity", "0.95") + .attr("stroke-width", "2") + .style("stroke", "gray"); var top_margin = 15; var overlay_header_height = 50; @@ -120,8 +129,8 @@ function hostnode_click(d) { d3.select(this) .attr("transform", "translate(" + d.x + "," + d.y + ")"); cur_node.select('#overlay_link') - .attr("x2", d.x + top_margin) - .attr("y2", d.y + 12); + .attr("x2", d.x + left_margin + 3) + .attr("y2", d.y + top_margin + 7); })); overlay_hostname.append('rect') @@ -139,6 +148,14 @@ function hostnode_click(d) { // Modal display var url = "/tree/hostname/" + d.data.uuid; d3.json(url, {credentials: 'same-origin'}).then(function(urls) { + overlay_hostname + .append('circle') + .attr('id', 'overlay_circle_' + d.data.uuid) + .attr('height', overlay_header_height) + .attr('cx', left_margin + 10) + .attr('cy', top_margin + 15) + .attr('r', 12); + overlay_hostname .append('text') .attr('id', 'overlay_close_' + d.data.uuid) @@ -147,12 +164,7 @@ function hostnode_click(d) { .attr('y', top_margin + 25) .style("font-size", overlay_header_height - 20) .text('\u2716') - .on("mouseover", function(d) { - d3.select(this).style("cursor", "pointer"); - }) - .on("mouseout", function(d) { - d3.select(this).style("cursor", "default"); - }) + .attr('cursor', 'pointer') .on("click", function() { main_svg.selectAll('#overlay_' + d.data.uuid).remove(); cur_node.select('#overlay_link').remove(); @@ -160,7 +172,7 @@ function hostnode_click(d) { ); overlay_hostname.append('line') - .attr('id', 'overlay_separator_' + d.data.uuid) + .attr('id', 'overlay_separator_header' + d.data.uuid) .style("stroke", "gray") .style('stroke-width', 2) .attr('x1', 15) @@ -176,23 +188,53 @@ function hostnode_click(d) { height_text = text_node.node().getBBox().height; icon_list(overlay_hostname, left_margin + 5, top_margin + height_text + overlay_header_height + (interval_entries * index)); }); + overlay_hostname.append('line') + .attr('id', 'overlay_separator_footer' + d.data.uuid) + .style("stroke", "gray") + .style('stroke-width', 2) + .attr('x1', 15) + .attr('y1', overlay_hostname.node().getBBox().height + 15) + .attr('x2', 500) + .attr('y2', overlay_hostname.node().getBBox().height); + + overlay_hostname + .append('text') + .attr('id', 'overlay_download_' + d.data.uuid) + .attr('height', overlay_header_height - 10) + .attr('x', left_margin) + .attr('y', overlay_hostname.node().getBBox().height + 40) + .style("font-size", overlay_header_height - 30) + .text('Download URLs as text') + .attr('cursor', 'pointer') + .on("click", function() { + var url = "/tree/hostname/" + d.data.uuid + '/text'; + d3.blob(url, {credentials: 'same-origin'}).then(function(data) { + saveAs(data, 'file.md'); + }); + }); + overlay_bbox = overlay_hostname.node().getBBox(); overlay_hostname.select('rect') - .attr('width', overlay_bbox.width + left_margin) - .attr('height', overlay_bbox.height + top_margin); + .attr('width', function() { + optimal_size = overlay_bbox.width + left_margin + return optimal_size < max_overlay_width ? optimal_size : max_overlay_width; + }) + .attr('height', overlay_bbox.height + 10); overlay_hostname.select('#overlay_close_' + d.data.uuid) - .attr('x', overlay_bbox.width); + .attr('x', overlay_hostname.select('rect').node().getBBox().width - left_margin); - overlay_hostname.select('#overlay_separator_' + d.data.uuid) - .attr('x2', overlay_bbox.width + left_margin + 15); + overlay_hostname.select('#overlay_separator_header' + d.data.uuid) + .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 14); + overlay_hostname.select('#overlay_separator_footer' + d.data.uuid) + .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 14); cur_node.select('#overlay_link') - .attr("x1", cur_node.x) - .attr("y1", cur_node.y) - .attr("x2", top_margin) - .attr("y2", 12); + .attr("x1", 10) + .attr("y1", 0) + .attr("x2", left_margin + 3) + .attr("y2", top_margin + 7); }); }; @@ -273,6 +315,8 @@ function text_entry(parent_svg, relative_x_pos, relative_y_pos, onclick_callback .style("font-size", "16px") .attr("stroke-width", ".2px") .style("opacity", .9) + .attr('cursor', 'pointer') + .attr("clip-path", "url(#textOverlay)") .text(function(d) { d.data.total_width = 0; // reset total_width to_display = d.data.name