Merge branch 'Th4nat0s-anonyme'

Fix #25

lookyloo/lookyloo.py

@@ -61,6 +61,8 @@ class Lookyloo():
             if not title:
                 title = '!! No title found !! '
         cache = {'uuid': uuid, 'title': title}
+        if (report_dir / 'no_index').exists():  # If the folders claims anonymity
+            cache['no_index'] = 1
         self.redis.hmset(str(report_dir), cache)
         self.redis.hset('lookup_dirs', uuid, str(report_dir))
 
@@ -135,7 +137,7 @@ class Lookyloo():
             return self.sanejs.sha512(sha512)
         return {'response': []}
 
-    def scrape(self, url, depth: int=1, user_agent: str=None, perma_uuid: str=None):
+    def scrape(self, url, depth: int=1, listing: bool=True, user_agent: str=None, perma_uuid: str=None):
         if not url.startswith('http'):
             url = f'http://{url}'
         items = crawl(self.splash_url, url, depth, user_agent=user_agent, log_enabled=True, log_level='INFO')
@@ -147,6 +149,8 @@ class Lookyloo():
         width = len(str(len(items)))
         dirpath = self.scrape_dir / datetime.now().isoformat()
         dirpath.mkdir()
+        if not listing:  # Write no_index marker
+            (dirpath / 'no_index').touch()
         for i, item in enumerate(items):
             harfile = item['har']
             png = base64.b64decode(item['png'])

website/web/__init__.py

@@ -52,7 +52,7 @@ def submit():
 @app.route('/scrape', methods=['GET', 'POST'])
 def scrape_web():
     if request.form.get('url'):
-        perma_uuid = lookyloo.scrape(request.form.get('url'), request.form.get('depth'))
+        perma_uuid = lookyloo.scrape(request.form.get('url'), request.form.get('depth'), request.form.get('listing'))
         return redirect(url_for('tree', tree_uuid=perma_uuid))
     return render_template('scrape.html')
 
@@ -146,8 +146,7 @@ def index():
     titles = []
     for report_dir in lookyloo.report_dirs:
         cached = lookyloo.report_cache(report_dir)
-        if not cached:
+        if not cached or 'no_index' in cached:
             continue
         titles.append((cached['uuid'], cached['title']))
-
     return render_template('index.html', titles=titles)

website/web/templates/scrape.html

@@ -21,6 +21,8 @@
                 <option value="8">8</option>
                 <option value="9">9</option>
             </select>
+            <label for="listing">Public</label>
+            <input type="checkbox" name="listing" checked="true"></input>
         </div>
         <button type="submit" class="btn btn-default">Scrape</button>
     </form>