chg: [sglinjection Phone] add tld statistic, fix phone regex

bin/Credential.py

@@ -59,6 +59,12 @@ if __name__ == "__main__":
         db=p.config.get("ARDB_TermCred", "db"),
         decode_responses=True)
 
+    server_statistics = redis.StrictRedis(
+        host=p.config.get("ARDB_Statistics", "host"),
+        port=p.config.getint("ARDB_Statistics", "port"),
+        db=p.config.getint("ARDB_Statistics", "db"),
+        decode_responses=True)
+
     criticalNumberToAlert = p.config.getint("Credential", "criticalNumberToAlert")
     minTopPassList = p.config.getint("Credential", "minTopPassList")
 
@@ -143,10 +149,11 @@ if __name__ == "__main__":
         #for searching credential in termFreq
         date = datetime.datetime.now().strftime("%Y%m")
         for cred in creds:
-            mail = cred.split('@')[-1]
+            mail = cred.split('@')[-1].split()[0]
+            faup.decode(mail)
             tld = faup.get()['tld']
             print(tld)
-            server_statistics.hincrby('credential_by_tld:'+date, tld, MX_values[1][mail])
+            server_statistics.hincrby('credential_by_tld:'+date, tld, 1)
 
             cred = cred.split('@')[0] #Split to ignore mail address
 

bin/LibInjection.py

@@ -12,6 +12,8 @@ It tries to identify SQL Injections with libinjection.
 """
 
 import time
+import datetime
+import redis
 import string
 import urllib.request
 import re
@@ -54,6 +56,12 @@ def analyse(url, path):
         msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
         p.populate_set_out(msg, 'Tags')
 
+        #statistics
+        tld = url_parsed['tld']
+        if tld is not None:
+            date = datetime.datetime.now().strftime("%Y%m")
+            server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
+
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
     # Port of the redis instance used by pubsublogger
@@ -70,6 +78,12 @@ if __name__ == '__main__':
     # Sent to the logging a description of the module
     publisher.info("Try to detect SQL injection with LibInjection")
 
+    server_statistics = redis.StrictRedis(
+        host=p.config.get("ARDB_Statistics", "host"),
+        port=p.config.getint("ARDB_Statistics", "port"),
+        db=p.config.getint("ARDB_Statistics", "db"),
+        decode_responses=True)
+
     faup = Faup()
 
     # Endless loop getting messages from the input queue

bin/Phone.py

@@ -11,7 +11,9 @@ It apply phone number regexes on paste content and warn if above a threshold.
 
 """
 
+import datetime
 import time
+import redis
 import re
 import phonenumbers
 from packages import Paste
@@ -23,8 +25,10 @@ def search_phone(message):
     paste = Paste.Paste(message)
     content = paste.get_p_content()
     # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
-    reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
-    reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})')
+    #reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
+    #reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})')
+    # use non capturing group
+    reg_phone = re.compile(r'(?:\+\d{1,4}(?:\(\d\))?\d?|0\d?)(?:\d{6,8}|(?:[-/\. ]{1}\(?\d{2,4}\)?){3,4})')
     # list of the regex results in the Paste, may be null
     results = reg_phone.findall(content)
 
@@ -45,17 +49,23 @@ def search_phone(message):
         for phone_number in results:
             try:
                 x = phonenumbers.parse(phone_number, None)
+                print(x)
                 country_code = x.country_code
                 if stats.get(country_code) is None:
                     stats[country_code] = 1
                 else:
                     stats[country_code] = stats[country_code] + 1
-            except:
+            except Exception as e:
+                #print(e)
                 pass
+
+        date = datetime.datetime.now().strftime("%Y%m")
         for country_code in stats:
+            print(country_code)
             if stats[country_code] > 4:
                 publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
 
+
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
     # Port of the redis instance used by pubsublogger
@@ -72,6 +82,13 @@ if __name__ == '__main__':
     # Sent to the logging a description of the module
     publisher.info("Run Phone module")
 
+    # ARDB #
+    server_statistics = redis.StrictRedis(
+        host=p.config.get("ARDB_Statistics", "host"),
+        port=p.config.getint("ARDB_Statistics", "port"),
+        db=p.config.getint("ARDB_Statistics", "db"),
+        decode_responses=True)
+
     # Endless loop getting messages from the input queue
     while True:
         # Get one message from the input queue

bin/SQLInjectionDetection.py

@@ -12,6 +12,8 @@ It test different possibility to makes some sqlInjection.
 """
 
 import time
+import datetime
+import redis
 import string
 import urllib.request
 import re
@@ -85,6 +87,13 @@ def analyse(url, path):
 
             msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
             p.populate_set_out(msg, 'Tags')
+
+            #statistics
+            tld = url_parsed['tld']
+            if tld is not None:
+                date = datetime.datetime.now().strftime("%Y%m")
+                server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
+
         else:
             print("Potential SQL injection:")
             print(urllib.request.unquote(url))
@@ -143,6 +152,12 @@ if __name__ == '__main__':
     # Sent to the logging a description of the module
     publisher.info("Try to detect SQL injection")
 
+    server_statistics = redis.StrictRedis(
+        host=p.config.get("ARDB_Statistics", "host"),
+        port=p.config.getint("ARDB_Statistics", "port"),
+        db=p.config.getint("ARDB_Statistics", "db"),
+        decode_responses=True)
+
     faup = Faup()
 
     # Endless loop getting messages from the input queue