From 5cc07c0203f94d39bdf01326a43349ceb4dd60c4 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Mon, 14 Mar 2016 20:40:06 +0100
Subject: [PATCH] A minimal caching module added to cache link or url from MISP

---
 REQUIREMENTS                     |  1 +
 modules/expansion/sourcecache.py | 31 +++++++++++++++++++++++++++++++
 tests/bodysourcecache.json       |  1 +
 tests/query-sourcecache.sh       |  1 +
 4 files changed, 34 insertions(+)
 create mode 100755 modules/expansion/sourcecache.py
 create mode 100644 tests/bodysourcecache.json
 create mode 100644 tests/query-sourcecache.sh

diff --git a/REQUIREMENTS b/REQUIREMENTS
index f60bfe6..9da1b81 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -1,3 +1,4 @@
 tornado
 dnspython3
 requests
+urlarchiver
diff --git a/modules/expansion/sourcecache.py b/modules/expansion/sourcecache.py
new file mode 100755
index 0000000..0466372
--- /dev/null
+++ b/modules/expansion/sourcecache.py
@@ -0,0 +1,31 @@
+import json
+from url_archiver import url_archiver
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['link'], 'output': ['link']}
+moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy', 'description': 'Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page.'}
+archive_path = '/tmp/'
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if request.get('link'):
+        tocache = request['link']
+        archiver = url_archiver.Archive(archive_path=archive_path)
+        archiver.fetch(url=tocache)
+        mispattributes['output'] = ['link']
+    else:
+        misperrors['error'] = "Link is missing"
+        return misperrors
+    r = {'results': [{'types': mispattributes['output'], 'values': tocache}]}
+    return r
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    return moduleinfo
diff --git a/tests/bodysourcecache.json b/tests/bodysourcecache.json
new file mode 100644
index 0000000..327394a
--- /dev/null
+++ b/tests/bodysourcecache.json
@@ -0,0 +1 @@
+{"module": "sourcecache", "link": "http://cve.circl.lu/" }
diff --git a/tests/query-sourcecache.sh b/tests/query-sourcecache.sh
new file mode 100644
index 0000000..db6f365
--- /dev/null
+++ b/tests/query-sourcecache.sh
@@ -0,0 +1 @@
+curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @bodysourcecache.json -X POST