diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index bb77f9dc..fa1c5f2e 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -61,6 +61,8 @@ class Lookyloo(): if not title: title = '!! No title found !! ' cache = {'uuid': uuid, 'title': title} + if (report_dir / 'no_index').exists(): # If the folders claims anonymity + cache['no_index'] = 1 self.redis.hmset(str(report_dir), cache) self.redis.hset('lookup_dirs', uuid, str(report_dir)) @@ -135,7 +137,7 @@ class Lookyloo(): return self.sanejs.sha512(sha512) return {'response': []} - def scrape(self, url, depth: int=1, user_agent: str=None, perma_uuid: str=None): + def scrape(self, url, depth: int=1, listing: bool=True, user_agent: str=None, perma_uuid: str=None): if not url.startswith('http'): url = f'http://{url}' items = crawl(self.splash_url, url, depth, user_agent=user_agent, log_enabled=True, log_level='INFO') @@ -147,6 +149,8 @@ class Lookyloo(): width = len(str(len(items))) dirpath = self.scrape_dir / datetime.now().isoformat() dirpath.mkdir() + if not listing: # Write no_index marker + (dirpath / 'no_index').touch() for i, item in enumerate(items): harfile = item['har'] png = base64.b64decode(item['png']) diff --git a/website/web/__init__.py b/website/web/__init__.py index 730da195..cedc33f2 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -52,7 +52,7 @@ def submit(): @app.route('/scrape', methods=['GET', 'POST']) def scrape_web(): if request.form.get('url'): - perma_uuid = lookyloo.scrape(request.form.get('url'), request.form.get('depth')) + perma_uuid = lookyloo.scrape(request.form.get('url'), request.form.get('depth'), request.form.get('listing')) return redirect(url_for('tree', tree_uuid=perma_uuid)) return render_template('scrape.html') @@ -146,8 +146,7 @@ def index(): titles = [] for report_dir in lookyloo.report_dirs: cached = lookyloo.report_cache(report_dir) - if not cached: + if not cached or 'no_index' in cached: continue titles.append((cached['uuid'], cached['title'])) - return render_template('index.html', titles=titles) diff --git a/website/web/templates/scrape.html b/website/web/templates/scrape.html index 6b3cbef8..3a9551fd 100644 --- a/website/web/templates/scrape.html +++ b/website/web/templates/scrape.html @@ -21,6 +21,8 @@ <option value="8">8</option> <option value="9">9</option> </select> + <label for="listing">Public</label> + <input type="checkbox" name="listing" checked="true"></input> </div> <button type="submit" class="btn btn-default">Scrape</button> </form>