From c714d88977783372c3b889f446632b2c0b3ca94b Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 6 Dec 2017 16:23:13 +0100 Subject: [PATCH 01/18] addition: Added start and terminate script + added test_geo script that test important function in the geo helper --- tests/start_framework.sh | 19 +++++++++ tests/terminate_framework.sh | 19 +++++++++ tests/test_config.cfg | 58 ++++++++++++++++++++++++++++ tests/test_geo.py | 75 ++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100755 tests/start_framework.sh create mode 100755 tests/terminate_framework.sh create mode 100644 tests/test_config.cfg create mode 100755 tests/test_geo.py diff --git a/tests/start_framework.sh b/tests/start_framework.sh new file mode 100755 index 0000000..fadf95e --- /dev/null +++ b/tests/start_framework.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" + +[ -z "$DASH_HOME" ] && echo "Needs the env var DASHENV. Run the script from the virtual environment." && exit 1; + +conf_dir="${DASH_HOME}/config/" +redis_dir="${DASH_HOME}/../redis/src/" +test_dir="${DASH_HOME}/tests/" +screenName="Misp-Dashboard-test" + +screen -dmS "$screenName" +sleep 0.1 +echo -e $GREEN"\t* Launching Redis servers"$DEFAULT +screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server --port 6260; read x' diff --git a/tests/terminate_framework.sh b/tests/terminate_framework.sh new file mode 100755 index 0000000..2d5a8c0 --- /dev/null +++ b/tests/terminate_framework.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" + +[ -z "$DASH_HOME" ] && echo "Needs the env var DASHENV. Run the script from the virtual environment." && exit 1; + +conf_dir="${DASH_HOME}/config/" +redis_dir="${DASH_HOME}/../redis/src/" +redis_dir="../../redis/src/" +test_dir="${DASH_HOME}/tests/" +screenName="Misp-Dashboard-test" + + +bash -c $redis_dir'redis-cli -p 6260 shutdown' +screen -S $screenName -X quit diff --git a/tests/test_config.cfg b/tests/test_config.cfg new file mode 100644 index 0000000..7750887 --- /dev/null +++ b/tests/test_config.cfg @@ -0,0 +1,58 @@ +[Dashboard] +#hours +graph_log_refresh_rate = 1 +#sec +rotation_wait_time = 30 +max_img_rotation = 10 +hours_spanned = 48 +zoomlevel = 15 +# [1->12] +size_dashboard_left_width = 5 +size_openStreet_pannel_perc = 55 +size_world_pannel_perc = 35 +item_to_plot = Attribute.category +fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]] +char_separator=|| + +[GEO] +#min +updateFrequency = 60 +zoomlevel = 11 +# ~meter +clusteringDistance = 10 + +[CONTRIB] +max_number_of_last_contributor = 10 +min_between_reload = 5 +additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"] + +[Log] +directory=logs +filename=logs.log + +[RedisGlobal] +host=localhost +port=6250 +#misp_web_url = http://192.168.56.50 +misp_web_url = https://misppriv.circl.lu +#zmq_url=tcp://192.168.56.50:50000 +zmq_url=tcp://localhost:9990 + +[RedisLIST] +db=3 +listName=bufferList + +[RedisLog] +db=0 +channel=1 +channelLastContributor = lastContributor +channelLastAwards = lastAwards + +[RedisMap] +db=1 +channelDisp=PicToDisplay +pathMaxMindDB=../data/GeoIP2-City_20171017/GeoIP2-City.mmdb +path_countrycode_to_coord_JSON=../data/country_code_lat_long.json + +[RedisDB] +db=2 diff --git a/tests/test_geo.py b/tests/test_geo.py new file mode 100755 index 0000000..498636f --- /dev/null +++ b/tests/test_geo.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3.5 +import configparser +import redis +import sys,os +import datetime +sys.path.append('..') + +configfile = 'test_config.cfg' +cfg = configparser.ConfigParser() +cfg.read(configfile) + +serv_redis_db = redis.StrictRedis( + host='localhost', + port=6260, + db=1) + +from helpers import geo_helper +geo_helper = geo_helper.Geo_helper(serv_redis_db, cfg) + +categ = 'Network Activity' + +def wipeRedis(): + serv_redis_db.flushall() + +def errorExit(): + wipeRedis() + sys.exit(1) + +def test(): + today = datetime.datetime.now() + # IP -> Coord + supposed_ip = '8.8.8.8' + geo_helper.getCoordFromIpAndPublish(supposed_ip, categ) + rep = geo_helper.getTopCoord(today) + excpected_result = [['{"lat": 37.751, "lon": -97.822}', 1.0]] + if rep != excpected_result: + print('ip to coord result not matching') + errorExit() + + # gethitmap + rep = geo_helper.getHitMap(today) + excpected_result = [['US', 1.0]] + if rep != excpected_result: + print('getHitMap result not matching') + errorExit() + + # getCoordsByRadius + rep = geo_helper.getCoordsByRadius(today, today, 0.000, 0.000, '1') + excpected_result = [] + if rep != excpected_result: + print('getCoordsByRadius result not matching') + errorExit() + + rep = geo_helper.getCoordsByRadius(today, today, 37.750, -97.821, '10') + excpected_result = [[['{"categ": "Network Activity", "value": "8.8.8.8"}'], [-97.82200008630753, 37.75100012475438]]] + if rep != excpected_result: + print('getCoordsByRadius result not matching') + errorExit() + + wipeRedis() + + + # Phone -> Coord + phoneNumber = '(+352) 247-82000' + geo_helper.getCoordFromPhoneAndPublish(phoneNumber, categ) + rep = geo_helper.getTopCoord(datetime.datetime.now())[0] + excpected_result = ['{"lat": "49.7500", "lon": "6.1667"}', 1.0] + if rep != excpected_result: + print('Phone to coord result not matching') + errorExit() + + +test() +wipeRedis() +print('Tests succeeded') From 84f495bba8daecdc18194d9de9c94dd6fb8f7db6 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 7 Dec 2017 09:48:07 +0100 Subject: [PATCH 02/18] Added trending test script --- tests/test_trendings.py | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 tests/test_trendings.py diff --git a/tests/test_trendings.py b/tests/test_trendings.py new file mode 100755 index 0000000..80f9cca --- /dev/null +++ b/tests/test_trendings.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3.5 +import configparser +import redis +import sys,os +import datetime, time +sys.path.append('..') + +configfile = 'test_config.cfg' +cfg = configparser.ConfigParser() +cfg.read(configfile) + +serv_redis_db = redis.StrictRedis( + host='localhost', + port=6260, + db=1) + +from helpers import trendings_helper +trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg) + + +def wipeRedis(): + serv_redis_db.flushall() + +def errorExit(): + sys.exit(1) + +def test(): + flag_error = True + today = datetime.datetime.now() + now = time.time + + # Events + event1 = 'test_event_1' + event2 = 'test_event_2' + trendings_helper.addTrendingEvent(event1, now()) + trendings_helper.addTrendingEvent(event1, now()+5) + trendings_helper.addTrendingEvent(event2, now()+10) + expected_result = [[int(now()), [[event1, 2.0], [event2, 1.0]]]] + rep = trendings_helper.getTrendingEvents(today, today) + if rep[0][1] != expected_result[0][1]: #ignore timestamps + print('getTrendingEvents result not matching') + flag_error = False + + # Tags + tag1 = {'id': 'tag1', 'colour': 'blue', 'name': 'tag1Name'} + tag2 = {'id': 'tag2', 'colour': 'red', 'name': 'tag2Name'} + trendings_helper.addTrendingTags([tag1], now()) + trendings_helper.addTrendingTags([tag1], now()+5) + trendings_helper.addTrendingTags([tag2], now()+10) + expected_result = [[int(now()), [[tag1, 2.0], [tag2, 1.0]]]] + rep = trendings_helper.getTrendingTags(today, today) + if rep[0][1] != expected_result[0][1]: #ignore timestamps + print('getTrendingTags result not matching') + flag_error = False + + # Sightings + trendings_helper.addSightings(now()) + trendings_helper.addSightings(now()) + trendings_helper.addFalsePositive(now()) + expected_result = [[1512636256, {'sightings': 2, 'false_positive': 1}]] + rep = trendings_helper.getTrendingSightings(today, today) + if rep[0][1] != expected_result[0][1]: #ignore timestamps + print('getTrendingSightings result not matching') + flag_error = False + + return flag_error + +wipeRedis() +if test(): + wipeRedis() + print('Trendings tests succeeded') +else: + wipeRedis() + errorExit() From 7f30131497258dcfe8e59a59c0757e30502fdfc0 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 7 Dec 2017 09:51:50 +0100 Subject: [PATCH 03/18] update: Reversed logic with errorFlag + Added it to test_geo --- tests/test_geo.py | 22 +++++++++++++--------- tests/test_trendings.py | 12 ++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/test_geo.py b/tests/test_geo.py index 498636f..b2634c3 100755 --- a/tests/test_geo.py +++ b/tests/test_geo.py @@ -23,11 +23,12 @@ def wipeRedis(): serv_redis_db.flushall() def errorExit(): - wipeRedis() sys.exit(1) def test(): + flag_error = False today = datetime.datetime.now() + # IP -> Coord supposed_ip = '8.8.8.8' geo_helper.getCoordFromIpAndPublish(supposed_ip, categ) @@ -35,27 +36,27 @@ def test(): excpected_result = [['{"lat": 37.751, "lon": -97.822}', 1.0]] if rep != excpected_result: print('ip to coord result not matching') - errorExit() + flag_error = True # gethitmap rep = geo_helper.getHitMap(today) excpected_result = [['US', 1.0]] if rep != excpected_result: print('getHitMap result not matching') - errorExit() + flag_error = True # getCoordsByRadius rep = geo_helper.getCoordsByRadius(today, today, 0.000, 0.000, '1') excpected_result = [] if rep != excpected_result: print('getCoordsByRadius result not matching') - errorExit() + flag_error = True rep = geo_helper.getCoordsByRadius(today, today, 37.750, -97.821, '10') excpected_result = [[['{"categ": "Network Activity", "value": "8.8.8.8"}'], [-97.82200008630753, 37.75100012475438]]] if rep != excpected_result: print('getCoordsByRadius result not matching') - errorExit() + flag_error = True wipeRedis() @@ -67,9 +68,12 @@ def test(): excpected_result = ['{"lat": "49.7500", "lon": "6.1667"}', 1.0] if rep != excpected_result: print('Phone to coord result not matching') - errorExit() + flag_error = True - -test() wipeRedis() -print('Tests succeeded') +if test(): + wipeRedis() + errorExit() +else: + wipeRedis() + print('Geo tests succeeded') diff --git a/tests/test_trendings.py b/tests/test_trendings.py index 80f9cca..4cce0e7 100755 --- a/tests/test_trendings.py +++ b/tests/test_trendings.py @@ -25,7 +25,7 @@ def errorExit(): sys.exit(1) def test(): - flag_error = True + flag_error = False today = datetime.datetime.now() now = time.time @@ -39,7 +39,7 @@ def test(): rep = trendings_helper.getTrendingEvents(today, today) if rep[0][1] != expected_result[0][1]: #ignore timestamps print('getTrendingEvents result not matching') - flag_error = False + flag_error = True # Tags tag1 = {'id': 'tag1', 'colour': 'blue', 'name': 'tag1Name'} @@ -51,7 +51,7 @@ def test(): rep = trendings_helper.getTrendingTags(today, today) if rep[0][1] != expected_result[0][1]: #ignore timestamps print('getTrendingTags result not matching') - flag_error = False + flag_error = True # Sightings trendings_helper.addSightings(now()) @@ -61,14 +61,14 @@ def test(): rep = trendings_helper.getTrendingSightings(today, today) if rep[0][1] != expected_result[0][1]: #ignore timestamps print('getTrendingSightings result not matching') - flag_error = False + flag_error = True return flag_error wipeRedis() if test(): wipeRedis() - print('Trendings tests succeeded') + errorExit() else: wipeRedis() - errorExit() + print('Trendings tests succeeded') From 2c3f17c4a9feb22cbc9490ca2a39f21b66cdfb0e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 7 Dec 2017 13:03:52 +0100 Subject: [PATCH 04/18] fix: return datetime instead of timestamp (to stay consistent) --- helpers/users_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/users_helper.py b/helpers/users_helper.py index 815a8e3..5dd9994 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -51,7 +51,7 @@ class Users_helper: keyname = "{}:{}".format(self.keyTimestamp, org) timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) if date is None: - to_return = [ t[1] for t in timestamps ] + to_return = [ datetime.datetime.fromtimestamp(float(t[1])) for t in timestamps ] else: to_return = [] for t in timestamps: @@ -165,7 +165,7 @@ class Users_helper: data.append(to_append) except KeyError: # no data data.append([0 for x in range(24)]) - # swap: punchcard day starts on monday + # swap: punchcard day starts on sunday data = [data[6]]+data[:6] return data From 139bd993a2ae8f36876d71381043d50735dc3d5a Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 7 Dec 2017 13:05:31 +0100 Subject: [PATCH 05/18] content: Added user test script --- tests/test_users.py | 120 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 tests/test_users.py diff --git a/tests/test_users.py b/tests/test_users.py new file mode 100755 index 0000000..61e7b41 --- /dev/null +++ b/tests/test_users.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3.5 +import configparser +import redis +import sys,os +import datetime, time +sys.path.append('..') + +configfile = 'test_config.cfg' +cfg = configparser.ConfigParser() +cfg.read(configfile) + +serv_redis_db = redis.StrictRedis( + host='localhost', + port=6260, + db=1) + +from helpers import users_helper +users_helper = users_helper.Users_helper(serv_redis_db, cfg) + + +def wipeRedis(): + serv_redis_db.flushall() + +def errorExit(): + sys.exit(1) + +# return if array are equal even if they are unordered +def checkArrayEquality(arr1, arr2): + temp = arr2[:] + for i in arr1: + if i in temp: + temp.remove(i) + else: + return False + return True + + +def test(): + flag_error = False + now = int(time.time()) + today = datetime.datetime.fromtimestamp(now) + twoDayAgo = today - datetime.timedelta(days=2) + DAY = 60*60*24 + org = 'TEST_ORG' + org2 = 'TEST_ORG2' + + # logged in dates + users_helper.add_user_login(now, org) + users_helper.add_user_login(now+5, org) + users_helper.add_user_login(now-DAY*2, org) + expected_result = [datetime.datetime.fromtimestamp(now-DAY*2), datetime.datetime.fromtimestamp(now+5), datetime.datetime.fromtimestamp(now)] + rep = users_helper.getDates(org) + + if not checkArrayEquality(rep, expected_result): + print('getDates result not matching for all dates') + flag_error = True + expected_result = [datetime.datetime.fromtimestamp(now+5), datetime.datetime.fromtimestamp(now)] + rep = users_helper.getDates(org, datetime.datetime.now()) + if not checkArrayEquality(rep, expected_result): + print('getDates result not matching for query 1') + flag_error = True + expected_result = [] + rep = users_helper.getDates(org, datetime.datetime.now()-datetime.timedelta(days=7)) + if not checkArrayEquality(rep, expected_result): + print('getDates result not matching for query 2') + flag_error = True + + # all logged orgs + users_helper.add_user_login(now, org2) + expected_result = [datetime.datetime.fromtimestamp(now+5), datetime.datetime.fromtimestamp(now), datetime.datetime.fromtimestamp(now)] + rep = users_helper.getUserLogins(datetime.datetime.now()) + if not checkArrayEquality(rep, expected_result): + print('getUserLogins result not matching') + flag_error = True + + # all logged in org + expected_result = [org, org2] + rep = users_helper.getAllLoggedInOrgs(datetime.datetime.fromtimestamp(now+5), prev_days=7) + if not checkArrayEquality(rep, expected_result): + print('getAllLoggedInOrgs result not matching') + flag_error = True + + # punchcard + expected_result = [ [0 for x in range(24)] for y in range(7)] + # set correct values + day = today.weekday() + hour = today.hour + expected_result[day][hour] = 3 + day = twoDayAgo.weekday() + hour = twoDayAgo.hour + expected_result[day][hour] = 1 + # swap: punchcard day starts on sunday + expected_result = [expected_result[6]]+expected_result[:6] + rep = users_helper.getUserLoginsForPunchCard(datetime.datetime.fromtimestamp(now), org=None, prev_days=6) + if not checkArrayEquality(rep, expected_result): + print('getUserLoginsForPunchCard result not matching') + flag_error = True + + # overtime + rep = users_helper.getUserLoginsAndContribOvertime(datetime.datetime.fromtimestamp(now), org=None, prev_days=6) + t1 = all([tab[1]==0 for tab in rep['contrib']]) # no contribution + t2 = [True for tab in rep['login'] if tab[1] == 3] + t2 = t2[0] and len(t2)==1 # one login at 3, others at 0 + if not (t1 and t2): + print('getUserLoginsAndContribOvertime result not matching') + flag_error = True + + + + + + return flag_error + +wipeRedis() +if test(): + wipeRedis() + errorExit() +else: + wipeRedis() + print('Users tests succeeded') From f5757362dba4c423fcf9d7a275a6b44a8bb2fc02 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 7 Dec 2017 13:23:22 +0100 Subject: [PATCH 06/18] fix: return flag_error value --- tests/test_geo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_geo.py b/tests/test_geo.py index b2634c3..94b9801 100755 --- a/tests/test_geo.py +++ b/tests/test_geo.py @@ -33,7 +33,7 @@ def test(): supposed_ip = '8.8.8.8' geo_helper.getCoordFromIpAndPublish(supposed_ip, categ) rep = geo_helper.getTopCoord(today) - excpected_result = [['{"lat": 37.751, "lon": -97.822}', 1.0]] + excpected_result = [['{"lat": 37.751, "lon": -97.822, "categ": "Network Activity", "value": "8.8.8.8"}', 1.0]] if rep != excpected_result: print('ip to coord result not matching') flag_error = True @@ -70,6 +70,8 @@ def test(): print('Phone to coord result not matching') flag_error = True + return flag_error + wipeRedis() if test(): wipeRedis() From ef6f9190b22d198de206562b73ce35210029a936 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 11 Dec 2017 08:58:51 +0100 Subject: [PATCH 07/18] update: conf + added prints --- tests/start_framework.sh | 2 +- tests/terminate_framework.sh | 2 ++ tests/test_config.cfg | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/start_framework.sh b/tests/start_framework.sh index fadf95e..25ce738 100755 --- a/tests/start_framework.sh +++ b/tests/start_framework.sh @@ -15,5 +15,5 @@ screenName="Misp-Dashboard-test" screen -dmS "$screenName" sleep 0.1 -echo -e $GREEN"\t* Launching Redis servers"$DEFAULT +echo -e $GREEN"* Launching Redis servers"$DEFAULT screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server --port 6260; read x' diff --git a/tests/terminate_framework.sh b/tests/terminate_framework.sh index 2d5a8c0..f9cf426 100755 --- a/tests/terminate_framework.sh +++ b/tests/terminate_framework.sh @@ -17,3 +17,5 @@ screenName="Misp-Dashboard-test" bash -c $redis_dir'redis-cli -p 6260 shutdown' screen -S $screenName -X quit +echo -e $GREEN"* Shutting down Redis servers"$DEFAULT + diff --git a/tests/test_config.cfg b/tests/test_config.cfg index 7750887..514f367 100644 --- a/tests/test_config.cfg +++ b/tests/test_config.cfg @@ -32,7 +32,7 @@ filename=logs.log [RedisGlobal] host=localhost -port=6250 +port=6260 #misp_web_url = http://192.168.56.50 misp_web_url = https://misppriv.circl.lu #zmq_url=tcp://192.168.56.50:50000 From 5de0d3181873a485e5d5170629d0e96dc3274367 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 11 Dec 2017 12:19:11 +0100 Subject: [PATCH 08/18] feature: Added draft of timeline which shows event overtime. The event currently span consecutive days (if it has been seen). --- helpers/trendings_helper.py | 40 +++++++++++++++++++++++++++++++++++++ install_dependencies.sh | 5 +++++ server.py | 12 +++++++++++ static/js/trendings.js | 29 ++++++++++++++++++++++++++- templates/trendings.html | 15 ++++++++++++++ 5 files changed, 100 insertions(+), 1 deletion(-) diff --git a/helpers/trendings_helper.py b/helpers/trendings_helper.py index b85caca..06951bb 100644 --- a/helpers/trendings_helper.py +++ b/helpers/trendings_helper.py @@ -158,3 +158,43 @@ class Trendings_helper: tagSet.add(tag['name']) to_ret[self.keyTag] = list(tagSet) return to_ret + + # In contrary of getGenericTrending, it regroups items in the format: {item, start: timestamp1, end: timestamp2} + # so that it can be displayed easily on the timeline. + def getGenericTrendingOvertime(self, dateS, dateE, trendingType=None, topNum=0): + trendingType = self.keyEvent + dico_items = {} + to_format = [] + prev_days = (dateE - dateS).days + # get data + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) + data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = data if data is not None else [] + to_format.append([util.getTimestamp(curDate), data]) + + for timestamp, array in to_format: + for item, _ in array: + if item not in dico_items: + dico_items[item] = [] + dico_items[item].append(timestamp) + + # sort timestamps in correct order + for item in dico_items.keys(): + dico_items[item].sort() + # dico_items have the form: {item: [t1,t2,t4], ...} + to_ret = [] + ONEDAY = 60*60*24 + for item, timestamps in dico_items.items(): + obj = {'name': item, 'start': timestamps[0]-ONEDAY, 'end': timestamps[0]} + for t in timestamps: + if t-obj['end'] > ONEDAY: #new entry + to_ret.append(obj) + obj['start'] = t-ONEDAY + obj['end'] = t + else: # contrinue entry + obj['end'] = t + to_ret.append(obj) + + return to_ret diff --git a/install_dependencies.sh b/install_dependencies.sh index 420fade..7c98f8b 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -108,4 +108,9 @@ mv temp/jquery-punchcard/src/punchcard.js ./static/js mv temp/jquery-punchcard/src/punchcard.css ./static/css wget https://momentjs.com/downloads/moment.js -O ./static/js/moment.js +# timeline +VISJS_VERSION="4.21.0" +https://cdnjs.cloudflare.com/ajax/libs/vis/${VISJS_VERSION}/vis.min.js +https://cdnjs.cloudflare.com/ajax/libs/vis/${VISJS_VERSION}/vis.min.css + rm -rf ./temp diff --git a/server.py b/server.py index ee57001..a23e40d 100755 --- a/server.py +++ b/server.py @@ -536,5 +536,17 @@ def getTypeaheadData(): data = trendings_helper.getTypeaheadData(dateS, dateE) return jsonify(data) +@app.route("/_getGenericTrendingOvertime") +def getGenericTrendingOvertime(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + data = trendings_helper.getGenericTrendingOvertime(dateS, dateE) + return jsonify(data) + if __name__ == '__main__': app.run(host='localhost', port=8001, threaded=True) diff --git a/static/js/trendings.js b/static/js/trendings.js index 79741e9..7d38216 100644 --- a/static/js/trendings.js +++ b/static/js/trendings.js @@ -9,6 +9,7 @@ var tagPie = ["#tagPie"]; var tagLine = ["#tagLine"]; var sightingLineWidget; var discLine = ["#discussionLine"]; +var timeline; var allData; var globalColorMapping = {}; @@ -103,6 +104,8 @@ var typeaheadOption_tag = { updateLineForLabel(tagLine, tag, undefined, url_getTrendingTag); } } +var timeline_option = {}; + /* FUNCTIONS */ function getColor(label) { @@ -396,6 +399,29 @@ function updateDisc() { }); } +function updateTimeline() { + $.getJSON( url_getGenericTrendingOvertime+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { + var items = []; + var i = 1; + for (var obj of data) { + if (obj.end == obj.start) { console.log(obj);} + items.push({ + id: i, + content: obj.name, + start: obj.start*1000, + end: obj.end*1000 + }); + i++; + } + items = new vis.DataSet(items); + if (timeline === undefined) { // create timeline + timeline = new vis.Timeline(document.getElementById('timeline'), items, timeline_option); + } else { // update + timeline.setItems(items); + } + }); +} + function dateChanged() { dateStart = datePickerWidgetStart.datepicker( "getDate" ); dateEnd = datePickerWidgetEnd.datepicker( "getDate" ); @@ -404,6 +430,7 @@ function dateChanged() { updatePieLine(tagPie, tagLine, url_getTrendingTag); updateSignthingsChart(); updateDisc(); + updateTimeline(); } $(document).ready(function () { @@ -426,6 +453,7 @@ $(document).ready(function () { updatePieLine(tagPie, tagLine, url_getTrendingTag) updateSignthingsChart(); updateDisc(); + updateTimeline(); $( "#num_selector" ).change(function() { var sel = parseInt($( this ).val()); @@ -437,5 +465,4 @@ $(document).ready(function () { position: "absolute", display: "none", }).appendTo("body"); - }); diff --git a/templates/trendings.html b/templates/trendings.html index 924e316..2a71b7e 100644 --- a/templates/trendings.html +++ b/templates/trendings.html @@ -36,6 +36,9 @@ + + +