From 9b3d338d3ca595636391720b5588eeb3041ce262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 Dec 2020 13:55:03 +0100 Subject: [PATCH] chg: Rename analisys->submission, add unique urls on stats graph --- lookyloo/lookyloo.py | 46 +++++++++++++++---------------- website/web/static/stats_graph.js | 13 ++++++--- website/web/templates/stats.html | 10 +++---- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index ed1d64f6..175ca257 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -997,7 +997,7 @@ class Lookyloo(): today = date.today() calendar_week = today.isocalendar()[1] - stats_dict = {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0} + stats_dict = {'submissions': 0, 'submissions_with_redirects': 0, 'redirects': 0} stats: Dict[int, Dict[int, Dict[str, Any]]] = {} weeks_stats: Dict[int, Dict] = {} @@ -1006,30 +1006,30 @@ class Lookyloo(): cache = self.capture_cache(uuid) if 'timestamp' not in cache: continue - date_analysis: datetime = datetime.fromisoformat(cache['timestamp'].rstrip('Z')) # type: ignore + date_submission: datetime = datetime.fromisoformat(cache['timestamp'].rstrip('Z')) # type: ignore - if date_analysis.year not in stats: - stats[date_analysis.year] = {} - if date_analysis.month not in stats[date_analysis.year]: - stats[date_analysis.year][date_analysis.month] = defaultdict(dict, **stats_dict) - stats[date_analysis.year][date_analysis.month]['uniq_urls'] = set() - stats[date_analysis.year][date_analysis.month]['analysis'] += 1 - stats[date_analysis.year][date_analysis.month]['uniq_urls'].add(cache['url']) + if date_submission.year not in stats: + stats[date_submission.year] = {} + if date_submission.month not in stats[date_submission.year]: + stats[date_submission.year][date_submission.month] = defaultdict(dict, **stats_dict) + stats[date_submission.year][date_submission.month]['uniq_urls'] = set() + stats[date_submission.year][date_submission.month]['submissions'] += 1 + stats[date_submission.year][date_submission.month]['uniq_urls'].add(cache['url']) if len(cache['redirects']) > 0: # type: ignore - stats[date_analysis.year][date_analysis.month]['analysis_with_redirects'] += 1 - stats[date_analysis.year][date_analysis.month]['redirects'] += len(cache['redirects']) # type: ignore - stats[date_analysis.year][date_analysis.month]['uniq_urls'].update(cache['redirects']) + stats[date_submission.year][date_submission.month]['submissions_with_redirects'] += 1 + stats[date_submission.year][date_submission.month]['redirects'] += len(cache['redirects']) # type: ignore + stats[date_submission.year][date_submission.month]['uniq_urls'].update(cache['redirects']) - if date_analysis.isocalendar()[1] >= calendar_week - 1: - if date_analysis.isocalendar()[1] not in weeks_stats: - weeks_stats[date_analysis.isocalendar()[1]] = defaultdict(dict, **stats_dict) - weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'] = set() - weeks_stats[date_analysis.isocalendar()[1]]['analysis'] += 1 - weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'].add(cache['url']) + if date_submission.isocalendar()[1] >= calendar_week - 1: + if date_submission.isocalendar()[1] not in weeks_stats: + weeks_stats[date_submission.isocalendar()[1]] = defaultdict(dict, **stats_dict) + weeks_stats[date_submission.isocalendar()[1]]['uniq_urls'] = set() + weeks_stats[date_submission.isocalendar()[1]]['submissions'] += 1 + weeks_stats[date_submission.isocalendar()[1]]['uniq_urls'].add(cache['url']) if len(cache['redirects']) > 0: # type: ignore - weeks_stats[date_analysis.isocalendar()[1]]['analysis_with_redirects'] += 1 - weeks_stats[date_analysis.isocalendar()[1]]['redirects'] += len(cache['redirects']) # type: ignore - weeks_stats[date_analysis.isocalendar()[1]]['uniq_urls'].update(cache['redirects']) + weeks_stats[date_submission.isocalendar()[1]]['submissions_with_redirects'] += 1 + weeks_stats[date_submission.isocalendar()[1]]['redirects'] += len(cache['redirects']) # type: ignore + weeks_stats[date_submission.isocalendar()[1]]['uniq_urls'].update(cache['redirects']) statistics: Dict[str, List] = {'weeks': [], 'years': []} for week_number in sorted(weeks_stats.keys()): @@ -1041,7 +1041,7 @@ class Lookyloo(): statistics['weeks'].append(week_stat) for year in sorted(stats.keys()): - year_stats: Dict[str, Union[int, List]] = {'year': year, 'months': [], 'yearly_analysis': 0, 'yearly_redirects': 0} + year_stats: Dict[str, Union[int, List]] = {'year': year, 'months': [], 'yearly_submissions': 0, 'yearly_redirects': 0} for month in sorted(stats[year].keys()): month_stats = stats[year][month] urls = month_stats.pop('uniq_urls') @@ -1050,7 +1050,7 @@ class Lookyloo(): month_stats['uniq_domains'] = len(uniq_domains(urls)) year_stats['months'].append(month_stats) # type: ignore - year_stats['yearly_analysis'] += month_stats['analysis'] + year_stats['yearly_submissions'] += month_stats['submissions'] year_stats['yearly_redirects'] += month_stats['redirects'] statistics['years'].append(year_stats) return statistics diff --git a/website/web/static/stats_graph.js b/website/web/static/stats_graph.js index 3580caff..f9ec7599 100644 --- a/website/web/static/stats_graph.js +++ b/website/web/static/stats_graph.js @@ -7,12 +7,17 @@ var height = 800; d3.json('/json/stats').then(json => { var datasets = [] json.years.forEach(year => { - var data_year = { label: `Analysis ${year.year}`, x: [], y: [] } + var submissions_year = { label: `Submissions ${year.year}`, x: [], y: [] } + var unique_urls_year = { label: `Unique URLs ${year.year}`, x: [], y: [] } year.months.forEach(month => { - data_year.x.push(month.month_number) - data_year.y.push(month.analysis) + submissions_year.x.push(month.month_number) + unique_urls_year.x.push(month.month_number) + + submissions_year.y.push(month.submissions) + unique_urls_year.y.push(month.uniq_urls) }); - datasets.push(data_year) + datasets.push(submissions_year) + datasets.push(unique_urls_year) }); var x_scale = d3.scaleLinear() diff --git a/website/web/templates/stats.html b/website/web/templates/stats.html index 075949b4..583a20bc 100644 --- a/website/web/templates/stats.html +++ b/website/web/templates/stats.html @@ -19,8 +19,8 @@ - {{ week['analysis'] }} - {{ week['analysis_with_redirects'] }} + {{ week['submissions'] }} + {{ week['submissions_with_redirects'] }} {{ week['redirects'] }} {{ week['uniq_urls'] }} {{ week['uniq_domains'] }} @@ -35,7 +35,7 @@ {% for year in stats['years'] %}

Year: {{ year['year'] }}

@@ -54,8 +54,8 @@ - {{ month['analysis'] }} - {{ month['analysis_with_redirects'] }} + {{ month['submissions'] }} + {{ month['submissions_with_redirects'] }} {{ month['redirects'] }} {{ month['uniq_urls'] }} {{ month['uniq_domains'] }}