diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index ed418432..d422a91d 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -925,6 +925,12 @@ def get_crawlers_stats_by_day(date, domain_type): 'down': r_crawler.scard(f'{domain_type}_down:{date}'), } +def get_crawlers_stats_by_month(domain_type, date=None): + stats = [] + for date in Date.get_month_dates(date=date): + stats.append(get_crawlers_stats_by_day(date, domain_type)) + return stats + def get_crawlers_stats(domain_type=None): stats = {} diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 6e58fb91..73ba1e2a 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import datetime +from calendar import monthrange from dateutil.rrule import rrule, MONTHLY from dateutil.relativedelta import relativedelta @@ -90,6 +91,14 @@ def get_current_week_day(): start = dt - datetime.timedelta(days=dt.weekday()) return start.strftime("%Y%m%d") +def get_month_dates(date=None): + if date: + date = convert_date_str_to_datetime(date) + else: + date = datetime.date.today() + num_days = monthrange(date.year, date.month)[1] + return [datetime.date(date.year, date.month, day).strftime("%Y%m%d") for day in range(1, num_days+1)] + def get_date_week_by_date(date): dt = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) start = dt - datetime.timedelta(days=dt.weekday()) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index b57f193d..b015b7c4 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -306,6 +306,16 @@ def crawlers_last_domains_json(): stats.append(crawlers.get_crawlers_stats_by_day(date, domain_type)) return jsonify(stats) +@crawler_splash.route('/crawlers/last/domains/monthly/json') +@login_required +@login_read_only +def crawlers_last_domains_monthly_json(): + domain_type = request.args.get('type') + if domain_type not in crawlers.get_crawler_all_types(): + return jsonify({'error': 'Invalid domain type'}), 400 + stats = crawlers.get_crawlers_stats_by_month(domain_type) + return jsonify(stats) + #### Domains #### diff --git a/var/www/templates/crawler/crawler_splash/last_crawled.html b/var/www/templates/crawler/crawler_splash/last_crawled.html index 582e45cb..eb756e08 100644 --- a/var/www/templates/crawler/crawler_splash/last_crawled.html +++ b/var/www/templates/crawler/crawler_splash/last_crawled.html @@ -102,6 +102,8 @@ +
+ @@ -142,9 +144,11 @@ $(document).ready(function(){ $('#date-range-to-input').val(s2); } }); - chart.stackBarChart =barchart_type_stack("{{ url_for('crawler_splash.crawlers_last_domains_json') }}?type={{type}}", 'id'); - chart.onResize(); + chart.stackBarChart = barchart_type_stack("{{ url_for('crawler_splash.crawlers_last_domains_json') }}?type={{type}}", 'barchart_type'); + chart.stackBarChartMonth = barchart_type_stack("{{ url_for('crawler_splash.crawlers_last_domains_monthly_json') }}?type={{type}}", 'barchart_type_month'); + + chart.onResize() $(window).on("resize", function() { chart.onResize(); }); @@ -174,7 +178,7 @@ function toggle_sidebar(){