From a57b8aeeb496e905859742d954c98d6a9111a2a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 29 Feb 2020 01:33:03 +0100
Subject: [PATCH] new: csse covid19 daily report importer

---
 examples/import_csse_covid19_daily.py | 65 +++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100755 examples/import_csse_covid19_daily.py

diff --git a/examples/import_csse_covid19_daily.py b/examples/import_csse_covid19_daily.py
new file mode 100755
index 0000000..1b4e7cf
--- /dev/null
+++ b/examples/import_csse_covid19_daily.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+from csv import DictReader
+from pymisp import MISPEvent, MISPOrganisation, PyMISP
+from datetime import datetime
+from dateutil.parser import parse
+import json
+from pymisp.tools import feed_meta_generator
+
+make_feed = False
+
+path = Path('/home/raphael/gits/COVID-19/csse_covid_19_data/csse_covid_19_daily_reports/')
+
+
+if make_feed:
+    org = MISPOrganisation()
+    org.name = 'CIRCL'
+    org.uuid = "55f6ea5e-2c60-40e5-964f-47a8950d210f"
+else:
+    from covid_key import url, key
+    misp = PyMISP(url, key)
+
+for p in path.glob('**/*.csv'):
+    d = datetime.strptime(p.name[:-4], '%m-%d-%Y').date()
+    event = MISPEvent()
+    event.info = f"[{d.isoformat()}] CSSE COVID-19 daily report"
+    event.date = d
+    if make_feed:
+        event.orgc = org
+    else:
+        e = misp.search(eventinfo=event.info, metadata=True, pythonify=True)
+        if e:
+            # Already added.
+            continue
+    with p.open() as f:
+        reader = DictReader(f)
+        for row in reader:
+            obj = event.add_object(name='covid19-csse-daily-report', standalone=False)
+            if 'Province/State' in row:
+                if row['Province/State']:
+                    obj.add_attribute('province-state', row['Province/State'])
+            elif '\ufeffProvince/State' in row:
+                if row['\ufeffProvince/State']:
+                    obj.add_attribute('province-state', row['\ufeffProvince/State'])
+            else:
+                print(p, row.keys())
+                raise Exception()
+            obj.add_attribute('country-region', row['Country/Region'])
+            obj.add_attribute('update', parse(row['Last Update']))
+            if row['Confirmed']:
+                obj.add_attribute('confirmed', int(row['Confirmed']))
+            if row['Deaths']:
+                obj.add_attribute('death', int(row['Deaths']))
+            if row['Recovered']:
+                obj.add_attribute('recovered', int(row['Recovered']))
+    if make_feed:
+        with (Path('output') / f'{event.uuid}.json').open('w') as _w:
+            json.dump(event.to_feed(), _w)
+    else:
+        misp.add_event(event)
+
+if make_feed:
+    feed_meta_generator(Path('output'))