CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'FafnerKeyZee-patch-1' into main

pull/134/head
Raphaël Vinot 2020-11-24 21:01:56 +01:00
parent 9cd3984587 d964549e56
commit 7197b9f55a
2 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lookyloo/lookyloo.py
View File

@ -5,7 +5,6 @@ import os
import base64 import base64
from collections import defaultdict, Counter from collections import defaultdict, Counter
from datetime import datetime, date, timedelta from datetime import datetime, date, timedelta
import calendar
from email.message import EmailMessage from email.message import EmailMessage
from io import BufferedIOBase, BytesIO from io import BufferedIOBase, BytesIO
import ipaddress import ipaddress
@ -997,13 +996,13 @@ class Lookyloo():
urls.append(to_append) urls.append(to_append)
return hostnode, urls return hostnode, urls
def get_stats(self) -> Dict[str, List]: def get_stats(self) -> Dict[str, Union[List, Dict]]:
stats: Dict[int, Dict[int, Dict[str, Any]]] = {} stats: Dict[int, Dict[int, Dict[str, Any]]] = {}
today = date.today() today = date.today()
calendar_week = today.isocalendar()[1] calendar_week = today.isocalendar()[1]
weeks_stats: Dict[int, Dict] = {calendar_week - 1: {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()}, weeks_stats: Dict[int, Dict] = {calendar_week - 1: {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()},
calendar_week: {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()}} calendar_week: {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()}}
statistics: Dict[str, List] = {'weeks': [], 'years': []} statistics: Dict[str, Union[List, Dict]] = {'weeks': [], 'years': {}}
for uuid in self.capture_uuids: for uuid in self.capture_uuids:
cache = self.capture_cache(uuid) cache = self.capture_cache(uuid)
if 'timestamp' not in cache: if 'timestamp' not in cache:
@ -1026,6 +1025,7 @@ class Lookyloo():
weeks_stats[date_analysis.isocalendar()[1]]['redirects'] += len(cache['redirects']) # type: ignore weeks_stats[date_analysis.isocalendar()[1]]['redirects'] += len(cache['redirects']) # type: ignore
weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'].update(cache['redirects']) weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'].update(cache['redirects'])
weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'].add(cache['url']) weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'].add(cache['url'])
for week_number, week_stat in weeks_stats.items(): for week_number, week_stat in weeks_stats.items():
week = {} week = {}
week['week'] = week_number week['week'] = week_number
@ -1034,16 +1034,15 @@ class Lookyloo():
week['redirects'] = week_stat['redirects'] week['redirects'] = week_stat['redirects']
week['uniq_urls'] = len(week_stat['uniq_urls']) week['uniq_urls'] = len(week_stat['uniq_urls'])
week['uniq_domains'] = len(uniq_domains(week_stat['uniq_urls'])) week['uniq_domains'] = len(uniq_domains(week_stat['uniq_urls']))
statistics['weeks'].append(week) statistics['weeks'].append(week) # type: ignore
for year, data in stats.items(): for year, data in stats.items():
years: Dict[str, Union[Dict, int]] = {} years: Dict[Union[int, str], Union[Dict, int]] = {}
years['year'] = year
yearly_analysis = 0 yearly_analysis = 0
yearly_redirects = 0 yearly_redirects = 0
for month in sorted(data.keys()): for month in sorted(data.keys()):
_stats = data[month] _stats = data[month]
mstats = {} mstats = {}
mstats['month'] = month
mstats['analysys'] = _stats['analysis'] mstats['analysys'] = _stats['analysis']
mstats['analysis_with_redirects'] = _stats['analysis_with_redirects'] mstats['analysis_with_redirects'] = _stats['analysis_with_redirects']
mstats['redirects'] = _stats['redirects'] mstats['redirects'] = _stats['redirects']
@ -1051,8 +1050,8 @@ class Lookyloo():
mstats['uniq_domains'] = len(uniq_domains(_stats['uniq_urls'])) mstats['uniq_domains'] = len(uniq_domains(_stats['uniq_urls']))
yearly_analysis += _stats['analysis'] yearly_analysis += _stats['analysis']
yearly_redirects += _stats['redirects'] yearly_redirects += _stats['redirects']
years[calendar.month_name[month]] = mstats years[month] = mstats
years['yearly_analysis'] = yearly_analysis years['yearly_analysis'] = yearly_analysis
years['yearly_redirects'] = yearly_redirects years['yearly_redirects'] = yearly_redirects
statistics['years'].append(years) statistics['years'][year] = years
return statistics return statistics

2
website/web/__init__.py
View File

@ -679,4 +679,4 @@ def json_hostname_info():
@app.route('/json/stats', methods=['GET']) @app.route('/json/stats', methods=['GET'])
def json_stats(): def json_stats():
to_return = lookyloo.get_stats() to_return = lookyloo.get_stats()
return jsonify(to_return) return Response(json.dumps(to_return), mimetype='application/json')