lookyloo/tools/stats.py

from lookyloo import Lookyloo
import calendar
import datetime
from urllib.parse import urlparse
from typing import Dict, Any, Union, Set, List

lookyloo = Lookyloo()

stats: Dict[Union[str, int], Any] = {}

today = datetime.date.today()
calendar_week = today.isocalendar()[1]
weeks_stats: Dict[int, Dict[str, Union[int, Set[str]]]] = \
    {calendar_week - 1: {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()},
     calendar_week: {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()}}


def uniq_domains(uniq_urls: List[str]) -> Set[str]:
    domains = set()
    for url in uniq_urls:
        splitted = urlparse(url)
        if splitted.hostname:
            domains.add(splitted.hostname)
    return domains


for cache in lookyloo.sorted_capture_cache():
    date = cache.timestamp
    if date.year not in stats:
        stats[date.year] = {}
    if date.month not in stats[date.year]:
        stats[date.year][date.month] = {'analysis': 0, 'analysis_with_redirects': 0, 'redirects': 0, 'uniq_urls': set()}
    stats[date.year][date.month]['analysis'] += 1
    if len(cache.redirects) > 0:
        stats[date.year][date.month]['analysis_with_redirects'] += 1
    stats[date.year][date.month]['redirects'] += len(cache.redirects)
    stats[date.year][date.month]['uniq_urls'].update(cache.redirects)
    stats[date.year][date.month]['uniq_urls'].add(cache.url)
    if date.isocalendar()[1] in weeks_stats:
        weeks_stats[date.isocalendar()[1]]['analysis'] += 1  # type: ignore
        if len(cache.redirects) > 0:
            weeks_stats[date.isocalendar()[1]]['analysis_with_redirects'] += 1  # type: ignore
        weeks_stats[date.isocalendar()[1]]['redirects'] += len(cache.redirects)  # type: ignore
        weeks_stats[date.isocalendar()[1]]['uniq_urls'].update(cache.redirects)  # type: ignore
        weeks_stats[date.isocalendar()[1]]['uniq_urls'].add(cache.url)  # type: ignore

print('Statistics for the last two weeks:')
for week_number, week_stat in weeks_stats.items():
    print(f'Week {week_number}:')
    print('    Number of analysis:', week_stat['analysis'])
    print('    Number of analysis with redirects:', week_stat['analysis_with_redirects'])
    print('    Number of redirects:', week_stat['redirects'])
    print('    Number of unique URLs:', len(week_stat['uniq_urls']))  # type: ignore
    d = uniq_domains(week_stat['uniq_urls'])  # type: ignore[arg-type]
    print('    Number of unique domains:', len(d))


for year, data in stats.items():
    print('Year:', year)
    yearly_analysis = 0
    yearly_redirects = 0
    for month in sorted(data.keys()):
        stats = data[month]
        print('   ', calendar.month_name[month])
        print("\tNumber of analysis :", stats['analysis'])
        print("\tNumber of analysis with redirects:", stats['analysis_with_redirects'])
        print("\tNumber of redirects :", stats['redirects'])
        print('\tNumber of unique URLs:', len(stats['uniq_urls']))
        domains = uniq_domains(stats['uniq_urls'])
        print('\tNumber of unique domains:', len(domains))
        yearly_analysis += stats['analysis']
        yearly_redirects += stats['redirects']

    print("    Sum analysis:", yearly_analysis)
    print("    Sum redirects:", yearly_redirects)