From bc1eab3520a605e5c059dc549acc3e345451ba70 Mon Sep 17 00:00:00 2001
From: kx499 <rubix399@protonmail.com>
Date: Tue, 28 Feb 2017 22:04:24 -0500
Subject: [PATCH 1/3] fixed spacing, addressed error handling for public api,
 added subdomains, and added context comment

---
 misp_modules/modules/expansion/virustotal.py | 199 +++++++++++--------
 1 file changed, 113 insertions(+), 86 deletions(-)

diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py
index a5ccd7d..63a8720 100755
--- a/misp_modules/modules/expansion/virustotal.py
+++ b/misp_modules/modules/expansion/virustotal.py
@@ -7,7 +7,8 @@ import os
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"],
-                  'output':['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", "authentihash", "filename"]
+                  'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep",
+                             "authentihash", "filename"]
                   }
 
 # possible module-types: 'expansion', 'hover' or both
@@ -17,7 +18,9 @@ moduleinfo = {'version': '2', 'author': 'Hannah Ward',
 
 # config fields that your code expects from the site admin
 moduleconfig = ["apikey", "event_limit"]
-limit = 5 #Default
+limit = 5  # Default
+comment = '%s: Enriched via VT'
+
 
 def handler(q=False):
     global limit
@@ -30,145 +33,169 @@ def handler(q=False):
     limit = int(q["config"].get("event_limit", 5))
 
     r = {"results": []}
-    
+
     if "ip-src" in q:
-      r["results"] += getIP(q["ip-src"], key)
+        r["results"] += getIP(q["ip-src"], key)
     if "ip-dst" in q:
-      r["results"] += getIP(q["ip-dst"], key)
+        r["results"] += getIP(q["ip-dst"], key)
     if "domain" in q:
-      r["results"] += getDomain(q["domain"], key)
+        r["results"] += getDomain(q["domain"], key)
     if 'hostname' in q:
-      r["results"] += getDomain(q['hostname'], key)
+        r["results"] += getDomain(q['hostname'], key)
     if 'md5' in q:
-      r["results"] += getHash(q['md5'], key)
+        r["results"] += getHash(q['md5'], key)
     if 'sha1' in q:
-      r["results"] += getHash(q['sha1'], key)
+        r["results"] += getHash(q['sha1'], key)
     if 'sha256' in q:
-      r["results"] += getHash(q['sha256'], key)
+        r["results"] += getHash(q['sha256'], key)
     if 'sha512' in q:
-      r["results"] += getHash(q['sha512'], key)
+        r["results"] += getHash(q['sha512'], key)
 
     uniq = []
     for res in r["results"]:
-      if res not in uniq:
-        uniq.append(res)
+        if res not in uniq:
+            uniq.append(res)
     r["results"] = uniq
     return r
 
-def getHash(hash, key, do_not_recurse = False):
+
+def getHash(hash, key, do_not_recurse=False):
     global limit
     toReturn = []
-    req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", 
-            params = { "allinfo":1, "apikey":key, 'resource': hash}
-                      ).json()
+    try:
+        req = requests.get("https://www.virustotal.com/vtapi/v2/file/report",
+                           params={"allinfo": 1, "apikey": key, 'resource': hash}
+                           ).json()
+    except:
+        return []
+
     if req["response_code"] == 0:
-      #Nothing found
-      return []
+        # Nothing found
+        return []
 
     toReturn += getMoreInfo(req, key)
     return toReturn
 
-def getIP(ip, key, do_not_recurse = False):
+
+def getIP(ip, key, do_not_recurse=False):
     global limit
     toReturn = []
-    req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", 
-                       params = {"ip":ip, "apikey":key}
-                      ).json()
+    try:
+        req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report",
+                           params={"ip": ip, "apikey": key}
+                           ).json()
+    except:
+        return []
+
     if req["response_code"] == 0:
-      #Nothing found
-      return []
-    
+        # Nothing found
+        return []
+
     if "resolutions" in req:
-      for res in req["resolutions"][:limit]:
-        toReturn.append( {"types":["domain"], "values":[res["hostname"]]})
-        #Pivot from here to find all domain info
-        if not do_not_recurse:
-          toReturn += getDomain(res["hostname"], key, True)
+        for res in req["resolutions"][:limit]:
+            toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment":comment % ip})
+            # Pivot from here to find all domain info
+            if not do_not_recurse:
+                toReturn += getDomain(res["hostname"], key, True)
 
     toReturn += getMoreInfo(req, key)
     return toReturn
-    
+
+
 def getDomain(domain, key, do_not_recurse=False):
     global limit
     toReturn = []
-    req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", 
-                       params = {"domain":domain, "apikey":key}
-                      ).json()
+    try:
+        req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report",
+                           params={"domain": domain, "apikey": key}
+                           ).json()
+    except:
+        return []
+
     if req["response_code"] == 0:
-      #Nothing found
-      return []
-    
+        # Nothing found
+        return []
+
     if "resolutions" in req:
-      for res in req["resolutions"][:limit]:
-        toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]})
-        #Pivot from here to find all info on IPs
-        if not do_not_recurse:
-          toReturn += getIP(res["ip_address"], key, True)
+        for res in req["resolutions"][:limit]:
+            toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment":comment % domain})
+            # Pivot from here to find all info on IPs
+            if not do_not_recurse:
+                toReturn += getIP(res["ip_address"], key, True)
+    if "subdomains" in req:
+        for subd in req["subdomains"]:
+            toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain})
     toReturn += getMoreInfo(req, key)
     return toReturn
 
+
 def findAll(data, keys):
-  a = []
-  if isinstance(data, dict):
-    for key in data.keys():
-      if key in keys:
-        a.append(data[key])
-      else:
-        if isinstance(data[key], (dict, list)):
-          a += findAll(data[key], keys)
-  if isinstance(data, list):
-    for i in data:
-      a += findAll(i, keys)  
-    
-  return a 
+    a = []
+    if isinstance(data, dict):
+        for key in data.keys():
+            if key in keys:
+                a.append(data[key])
+            else:
+                if isinstance(data[key], (dict, list)):
+                    a += findAll(data[key], keys)
+    if isinstance(data, list):
+        for i in data:
+            a += findAll(i, keys)
+
+    return a
+
 
 def getMoreInfo(req, key):
     global limit
     r = []
-    #Get all hashes first
+    # Get all hashes first
     hashes = []
     hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"])
-    r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes})
+    r.append({"types": ["md5", "sha1", "sha256", "sha512"], "values": hashes})
     for hsh in hashes[:limit]:
-      #Search VT for some juicy info
-      data = requests.get("http://www.virustotal.com/vtapi/v2/file/report",
-                          params={"allinfo":1, "apikey":key, "resource":hsh}
-                         ).json()
+        # Search VT for some juicy info
+        try:
+            data = requests.get("http://www.virustotal.com/vtapi/v2/file/report",
+                                params={"allinfo": 1, "apikey": key, "resource": hsh}
+                                ).json()
+        except:
+            continue
 
-      # Go through each key and check if it exists
-      if "submission_names" in data:
-        r.append({'types':["filename"], "values":data["submission_names"]})
+        # Go through each key and check if it exists
+        if "submission_names" in data:
+            r.append({'types': ["filename"], "values": data["submission_names"], "comment":comment % hsh})
 
-      if "ssdeep" in data:
-        r.append({'types':["ssdeep"], "values":[data["ssdeep"]]})
+        if "ssdeep" in data:
+            r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment":comment % hsh})
 
-      if "authentihash" in data:
-        r.append({"types":["authentihash"], "values":[data["authentihash"]]})
+        if "authentihash" in data:
+            r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment":comment % hsh})
 
-      if "ITW_urls" in data:
-        r.append({"types":["url"], "values":data["ITW_urls"]})
+        if "ITW_urls" in data:
+            r.append({"types": ["url"], "values": data["ITW_urls"], "comment":comment % hsh})
 
-      #Get the malware sample
-      sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download",
-                            params = {"hash":hsh, "apikey":key})
-      
-      malsample = sample.content
+        # Get the malware sample
+        sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download",
+                              params={"hash": hsh, "apikey": key})
+
+        malsample = sample.content
+
+        # It is possible for VT to not give us any submission names
+        if "submission_names" in data:
+            r.append({"types": ["malware-sample"],
+                      "categories": ["Payload delivery"],
+                      "values": data["submission_names"],
+                      "data": str(base64.b64encode(malsample), 'utf-8')
+                      }
+                     )
 
-      # It is possible for VT to not give us any submission names
-      if "submission_names" in data:
-        r.append({"types":["malware-sample"], 
-                  "categories":["Payload delivery"],
-                  "values":data["submission_names"],
-                  "data": str(base64.b64encode(malsample), 'utf-8')
-                  }
-                )
-    
     return r
 
+
 def introspection():
     return mispattributes
 
+
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
-

From 01fdf3e52b63d0ab1d8d9b8af12234a71fc4f635 Mon Sep 17 00:00:00 2001
From: kx499 <rubix399@protonmail.com>
Date: Fri, 3 Mar 2017 15:55:52 -0500
Subject: [PATCH 2/3] Initial commit of IPRep module

---
 misp_modules/modules/expansion/iprep.py | 85 +++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100755 misp_modules/modules/expansion/iprep.py

diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py
new file mode 100755
index 0000000..1dc6460
--- /dev/null
+++ b/misp_modules/modules/expansion/iprep.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+
+import json
+import requests
+
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
+moduleinfo = {'version': '1.0', 'author': 'Keith Faber',
+              'description': 'Query IPRep Data for IP Address',
+              'module-type': ['expansion']}
+
+moduleconfig = ['apikey']
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if request.get('ip-src'):
+        toquery = request['ip-src']
+    elif request.get('ip-dst'):
+        toquery = request['ip-dst']
+    else:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+
+    if not request.get('config') and not request['config'].get('apikey'):
+        misperrors['error'] = 'IPRep api key is missing'
+        return misperrors
+
+    err, rep = parse_iprep(toquery, request['config'].get('apikey'))
+    if len(err) > 0:
+        misperrors['error'] = ','.join(err)
+        return misperrors
+    return rep
+
+
+def parse_iprep(ip, api):
+    meta_fields = ['origin', 'Query_Time', 'created_on', 'IP_Lookup_History', 'IPs_in_collection', '_id', 'disclaimer',
+                   'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result']
+    rep = []
+    err = []
+    url = 'https://www.packetmail.net/iprep.php/%s' % ip
+    try:
+        data = requests.get(url, params={'apikey': api}).json()
+    except:
+        return ['Error pulling data'], rep
+    # print '%s' % data
+    for name, val in data.iteritems():
+        if name not in meta_fields:
+            try:
+                context = val['context']
+                if type(context) is list:
+                    if context[0].get('alert'):
+                        context = ','.join([hit['alert']['signature'] for hit in context])
+                    elif context[0].get('signature'):
+                        context = ','.join([hit['signature'] for hit in context])
+                    elif context[0].get('target_port') and context[0].get('protocol'):
+                        context = ','.join(
+                            ['Port Attacked: %s %s' % (hit['target_port'], hit['protocol']) for hit in context])
+                    elif context[0].get('phishing_kit') and context[0].get('url'):
+                        context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context])
+                    else:
+                        context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].iteritems()])
+
+                if val.get('special_note'):
+                    context += '; ' + val['special_note']
+
+                misp_val = context
+                misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
+                rep.append({'types': mispattributes['output'], 'values': misp_val, 'comment': misp_comment})
+            except:
+                err.append('Error parsing source: %s' % name)
+
+    return err, rep
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

From 3ecd095d1e8e1879001eb0e4c9c2b5d020ef8745 Mon Sep 17 00:00:00 2001
From: kx499 <rubix399@protonmail.com>
Date: Sat, 4 Mar 2017 03:10:45 +0100
Subject: [PATCH 3/3] bug fixes, tweaks, and python3 learning curve :)

---
 misp_modules/modules/expansion/__init__.py |  2 +-
 misp_modules/modules/expansion/iprep.py    | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index 914cb1c..0fa3791 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -2,4 +2,4 @@ from . import _vmray
 
 __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
            'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache',
-           'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki']
+           'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep']
diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py
index 1dc6460..6073052 100755
--- a/misp_modules/modules/expansion/iprep.py
+++ b/misp_modules/modules/expansion/iprep.py
@@ -5,7 +5,7 @@ import requests
 
 
 misperrors = {'error': 'Error'}
-mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
+mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']}
 moduleinfo = {'version': '1.0', 'author': 'Keith Faber',
               'description': 'Query IPRep Data for IP Address',
               'module-type': ['expansion']}
@@ -33,7 +33,7 @@ def handler(q=False):
     if len(err) > 0:
         misperrors['error'] = ','.join(err)
         return misperrors
-    return rep
+    return {'results': rep}
 
 
 def parse_iprep(ip, api):
@@ -41,13 +41,14 @@ def parse_iprep(ip, api):
                    'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result']
     rep = []
     err = []
+    full_text = ''
     url = 'https://www.packetmail.net/iprep.php/%s' % ip
     try:
         data = requests.get(url, params={'apikey': api}).json()
     except:
         return ['Error pulling data'], rep
     # print '%s' % data
-    for name, val in data.iteritems():
+    for name, val in data.items():
         if name not in meta_fields:
             try:
                 context = val['context']
@@ -62,17 +63,19 @@ def parse_iprep(ip, api):
                     elif context[0].get('phishing_kit') and context[0].get('url'):
                         context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context])
                     else:
-                        context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].iteritems()])
+                        context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].items()])
 
                 if val.get('special_note'):
                     context += '; ' + val['special_note']
 
                 misp_val = context
+                full_text += '\n%s' % context
                 misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen'])
-                rep.append({'types': mispattributes['output'], 'values': misp_val, 'comment': misp_comment})
+                rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment})
             except:
                 err.append('Error parsing source: %s' % name)
 
+    rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'})
     return err, rep