new: allow to download the URLs list in a text file

lookyloo/__init__.py

@@ -62,7 +62,7 @@ def scrape():
     if request.form.get('url'):
         url = request.form.get('url')
         if not url.startswith('http'):
-            url = 'http://{}'.format(url)
+            url = f'http://{url}'
         depth = request.form.get('depth')
         if depth is None:
             depth = 1
@@ -98,6 +98,24 @@ def get_report_dirs():
     return sorted(HAR_DIR.iterdir(), reverse=True)
 
 
+@app.route('/tree/hostname/<node_uuid>/text', methods=['GET'])
+def hostnode_details_text(node_uuid):
+    with open(session["tree"], 'rb') as f:
+        ct = pickle.load(f)
+    hostnode = ct.root_hartree.get_host_node_by_uuid(node_uuid)
+    urls = []
+    for url in hostnode.urls:
+        urls.append(url.name)
+    content = '''# URLs
+
+{}
+'''.format('\n'.join(urls))
+    to_return = BytesIO(content.encode())
+    to_return.seek(0)
+    return send_file(to_return, mimetype='text/markdown',
+                     as_attachment=True, attachment_filename='file.md')
+
+
 @app.route('/tree/hostname/<node_uuid>', methods=['GET'])
 def hostnode_details(node_uuid):
     with open(session["tree"], 'rb') as f:
@@ -114,16 +132,18 @@ def urlnode_details(node_uuid):
     with open(session["tree"], 'rb') as f:
         ct = pickle.load(f)
     urlnode = ct.root_hartree.get_url_node_by_uuid(node_uuid)
-
     to_return = BytesIO()
+    got_content = False
     if hasattr(urlnode, 'body'):
-        with ZipFile(to_return, 'a', ZIP_DEFLATED, False) as zfile:
-            zfile.writestr(urlnode.filename, urlnode.body.getvalue())
-        to_return.seek(0)
-    # return send_file(urlnode.body, mimetype='application/zip',
-    #                 as_attachment=True, attachment_filename='file.zip')
-    with open('foo.bin', 'wb') as f:
-        f.write(to_return.getvalue())
+        body_content = urlnode.body.getvalue()
+        if body_content:
+            got_content = True
+            with ZipFile(to_return, 'w', ZIP_DEFLATED) as zfile:
+                zfile.writestr(urlnode.filename, urlnode.body.getvalue())
+    if not got_content:
+        with ZipFile(to_return, 'w', ZIP_DEFLATED) as zfile:
+            zfile.writestr('file.txt', b'Response body empty')
+    to_return.seek(0)
     return send_file(to_return, mimetype='application/zip',
                      as_attachment=True, attachment_filename='file.zip')
 

lookyloo/static/tree.js

@@ -6,12 +6,19 @@ var margin = {top: 20, right: 200, bottom: 30, left: 90},
     height = 10000 - margin.top - margin.bottom;
 
 var node_width = 0;
+var max_overlay_width = 1500;
 var node_height = 45;
 
 var main_svg = d3.select("body").append("svg")
             .attr("width", width + margin.right + margin.left)
             .attr("height", height + margin.top + margin.bottom)
 
+main_svg.append("clipPath")
+    .attr("id", "textOverlay")
+    .append("rect")
+    .attr('width', max_overlay_width - 25)
+    .attr('height', node_height);
+
 // Add background pattern
 var pattern = main_svg.append("defs").append('pattern')
     .attr('id', 'backstripes')
@@ -75,7 +82,7 @@ function getBB(selection) {
 };
 
 function urlnode_click(d) {
-    var url = "url/" + d.data.uuid;
+    var url = "tree/url/" + d.data.uuid;
     d3.blob(url, {credentials: 'same-origin'}).then(function(data) {
         saveAs(data, 'file.zip');
     });
@@ -102,7 +109,9 @@ function hostnode_click(d) {
 
     cur_node.append('line')
                 .attr('id', 'overlay_link')
-                .style("stroke", "black");
+                .style("opacity", "0.95")
+                .attr("stroke-width", "2")
+                .style("stroke", "gray");
 
     var top_margin = 15;
     var overlay_header_height = 50;
@@ -120,8 +129,8 @@ function hostnode_click(d) {
             d3.select(this)
                 .attr("transform", "translate(" + d.x + "," + d.y + ")");
             cur_node.select('#overlay_link')
-                .attr("x2", d.x + top_margin)
-                .attr("y2", d.y + 12);
+                .attr("x2", d.x + left_margin + 3)
+                .attr("y2", d.y + top_margin + 7);
         }));
 
     overlay_hostname.append('rect')
@@ -139,6 +148,14 @@ function hostnode_click(d) {
     // Modal display
     var url = "/tree/hostname/" + d.data.uuid;
     d3.json(url, {credentials: 'same-origin'}).then(function(urls) {
+        overlay_hostname
+            .append('circle')
+            .attr('id', 'overlay_circle_' + d.data.uuid)
+            .attr('height', overlay_header_height)
+            .attr('cx', left_margin + 10)
+            .attr('cy', top_margin + 15)
+            .attr('r', 12);
+
         overlay_hostname
             .append('text')
             .attr('id', 'overlay_close_' + d.data.uuid)
@@ -147,12 +164,7 @@ function hostnode_click(d) {
             .attr('y', top_margin + 25)
             .style("font-size", overlay_header_height - 20)
             .text('\u2716')
-            .on("mouseover", function(d) {
-                    d3.select(this).style("cursor", "pointer");
-                })
-            .on("mouseout", function(d) {
-                    d3.select(this).style("cursor", "default");
-                })
+            .attr('cursor', 'pointer')
             .on("click", function() {
                     main_svg.selectAll('#overlay_' + d.data.uuid).remove();
                     cur_node.select('#overlay_link').remove();
@@ -160,7 +172,7 @@ function hostnode_click(d) {
             );
 
         overlay_hostname.append('line')
-            .attr('id', 'overlay_separator_' + d.data.uuid)
+            .attr('id', 'overlay_separator_header' + d.data.uuid)
             .style("stroke", "gray")
             .style('stroke-width', 2)
             .attr('x1', 15)
@@ -176,23 +188,53 @@ function hostnode_click(d) {
             height_text = text_node.node().getBBox().height;
             icon_list(overlay_hostname, left_margin + 5, top_margin + height_text + overlay_header_height + (interval_entries * index));
         });
+        overlay_hostname.append('line')
+            .attr('id', 'overlay_separator_footer' + d.data.uuid)
+            .style("stroke", "gray")
+            .style('stroke-width', 2)
+            .attr('x1', 15)
+            .attr('y1', overlay_hostname.node().getBBox().height + 15)
+            .attr('x2', 500)
+            .attr('y2', overlay_hostname.node().getBBox().height);
+
+        overlay_hostname
+            .append('text')
+            .attr('id', 'overlay_download_' + d.data.uuid)
+            .attr('height', overlay_header_height - 10)
+            .attr('x', left_margin)
+            .attr('y', overlay_hostname.node().getBBox().height + 40)
+            .style("font-size", overlay_header_height - 30)
+            .text('Download URLs as text')
+            .attr('cursor', 'pointer')
+            .on("click", function() {
+                var url = "/tree/hostname/" + d.data.uuid + '/text';
+                d3.blob(url, {credentials: 'same-origin'}).then(function(data) {
+                    saveAs(data, 'file.md');
+                });
+            });
+
         overlay_bbox = overlay_hostname.node().getBBox();
         overlay_hostname.select('rect')
-            .attr('width', overlay_bbox.width + left_margin)
-            .attr('height', overlay_bbox.height + top_margin);
+            .attr('width', function() {
+                optimal_size = overlay_bbox.width + left_margin
+                return optimal_size < max_overlay_width ? optimal_size : max_overlay_width;
+            })
+            .attr('height', overlay_bbox.height + 10);
 
         overlay_hostname.select('#overlay_close_' + d.data.uuid)
-            .attr('x', overlay_bbox.width);
+            .attr('x', overlay_hostname.select('rect').node().getBBox().width - left_margin);
 
-        overlay_hostname.select('#overlay_separator_' + d.data.uuid)
-            .attr('x2', overlay_bbox.width + left_margin + 15);
+        overlay_hostname.select('#overlay_separator_header' + d.data.uuid)
+            .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 14);
+        overlay_hostname.select('#overlay_separator_footer' + d.data.uuid)
+            .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 14);
 
 
         cur_node.select('#overlay_link')
-                    .attr("x1", cur_node.x)
-                    .attr("y1", cur_node.y)
-                    .attr("x2", top_margin)
-                    .attr("y2", 12);
+                    .attr("x1", 10)
+                    .attr("y1", 0)
+                    .attr("x2", left_margin + 3)
+                    .attr("y2", top_margin + 7);
     });
 };
 
@@ -273,6 +315,8 @@ function text_entry(parent_svg, relative_x_pos, relative_y_pos, onclick_callback
           .style("font-size", "16px")
           .attr("stroke-width", ".2px")
           .style("opacity", .9)
+          .attr('cursor', 'pointer')
+          .attr("clip-path", "url(#textOverlay)")
           .text(function(d) {
               d.data.total_width = 0; // reset total_width
               to_display = d.data.name