Merge branch 'iss160' into showDiff

Preemptively preventing merge-conflict with branch iss160 (moving to
configuration)

bin/Categ.py

@@ -52,6 +52,7 @@ if __name__ == "__main__":
     config_section = 'Categ'
 
     p = Process(config_section)
+    matchingThreshold = p.config.getint("Categ", "matchingThreshold")
 
     # SCRIPT PARSER #
     parser = argparse.ArgumentParser(description='Start Categ module on files.')
@@ -90,7 +91,7 @@ if __name__ == "__main__":
 
         for categ, pattern in tmp_dict.items():
             found = set(re.findall(pattern, content))
-            if len(found) > 0:
+            if len(found) >= matchingThreshold:
                 msg = '{} {}'.format(paste.p_path, len(found))
                 print msg, categ
                 p.populate_set_out(msg, categ)

bin/Credential.py

@@ -41,7 +41,6 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
 REDIS_KEY_ALL_PATH_SET = 'AllPath'
 REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
 REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
-MINIMUMSIZETHRESHOLD = 3
 
 if __name__ == "__main__":
     publisher.port = 6380
@@ -50,13 +49,16 @@ if __name__ == "__main__":
     p = Process(config_section)
     publisher.info("Find credentials")
     
+    minimumLengthThreshold = p.config.getint("Credential", "minimumLengthThreshold")
+
     faup = Faup()
     server_cred = redis.StrictRedis(
         host=p.config.get("Redis_Level_DB_TermCred", "host"),
         port=p.config.get("Redis_Level_DB_TermCred", "port"),
         db=p.config.get("Redis_Level_DB_TermCred", "db"))
 
-    critical = 8
+    criticalNumberToAlert = p.config.getint("Credential", "criticalNumberToAlert")
+    minTopPassList = p.config.getint("Credential", "minTopPassList")
 
     regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
     regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
@@ -71,7 +73,7 @@ if __name__ == "__main__":
 
         filepath, count = message.split()
 
-        if count < 5:
+        if count < minTopPassList:
             # Less than 5 matches from the top password list, false positive.
             print("false positive:", count)
             continue
@@ -94,7 +96,7 @@ if __name__ == "__main__":
         print('\n '.join(creds))
 
         #num of creds above tresh, publish an alert
-        if len(creds) > critical:
+        if len(creds) > criticalNumberToAlert:
             print("========> Found more than 10 credentials in this file : {}".format(filepath))
             publisher.warning(to_print)
             #Send to duplicate
@@ -154,6 +156,6 @@ if __name__ == "__main__":
             #Add the split to redis, each split point towards its initial credential unique number
             splitedCred = re.findall(REGEX_CRED, cred)
             for partCred in splitedCred:
-                if len(partCred) > MINIMUMSIZETHRESHOLD:
+                if len(partCred) > minimumLengthThreshold:
                     server_cred.sadd(partCred, uniq_num_cred)
 

bin/Mixer.py

@@ -21,6 +21,7 @@ Depending on the configuration, this module will process the feed as follow:
 
     operation_mode 3: "Don't look if duplicated content"
         - SImply do not bother to check if it is a duplicate
+        - Simply do not bother to check if it is a duplicate
 
 Note that the hash of the content is defined as the sha1(gzip64encoded).
 

bin/packages/config.cfg.sample

@@ -32,6 +32,18 @@ minute_processed_paste = 10
 DiffMaxLineLength = 10000
 
 #### Modules ####
+[Categ]
+#Minimum number of match between the paste and the category file
+matchingThreshold=1
+
+[Credential]
+#Minimum length that a credential must have to be considered as such
+minimumLengthThreshold=3
+#Will be pushed as alert if the number of credentials is greater to that number
+criticalNumberToAlert=8
+#Will be considered as false positive if less that X matches from the top password list
+minTopPassList=5
+
 [Modules_Duplicates]
 #Number of month to look back
 maximum_month_range = 3
@@ -47,8 +59,8 @@ min_paste_size = 0.3
 threshold_stucked_module=600
 
 [Module_Mixer]
-#Define the configuration of the mixer, possible value: 1 or 2
-operation_mode = 1
+#Define the configuration of the mixer, possible value: 1, 2 or 3
+operation_mode = 3
 #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
 ttl_duplicate = 86400
 
@@ -141,7 +153,7 @@ maxDuplicateToPushToMISP=10
 # e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
 [ZMQ_Global]
 #address = tcp://crf.circl.lu:5556
-address = tcp://127.0.0.1:5556
+address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
 channel = 102
 bind = tcp://127.0.0.1:5556