Added script module-dataflow-graph + fixed bug in module.cfg preventing the credential module to receive data.

bin/packages/modules.cfg

@@ -34,7 +34,7 @@ subscribe = Redis_Global
 publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
 
 [CreditCards]
-subscribe = Redis_CreditCard
+subscribe = Redis_CreditCards
 publish = Redis_Duplicate,Redis_ModuleStats,Redis_BrowseWarningPaste
 
 [Mail]

doc/generate_graph_data.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+content = ""
+modules = {}
+all_modules = []
+curr_module = ""
+streamingPub = {}
+streamingSub = {}
+
+with open('../bin/packages/modules.cfg', 'r') as f:
+    for line in f:
+        if line[0] != '#':
+            if line[0] == '[':
+                curr_name = line.replace('[','').replace(']','').replace('\n', '').replace(' ', '')
+                all_modules.append(curr_name)
+                modules[curr_name] = {'sub': [], 'pub': []}
+                curr_module = curr_name
+            elif curr_module != "": # searching for sub or pub
+                if line.startswith("subscribe"):
+                    curr_subscribers = [w for w in line.replace('\n', '').replace(' ', '').split('=')[1].split(',')]
+                    modules[curr_module]['sub'] = curr_subscribers
+                    for sub in curr_subscribers:
+                        streamingSub[sub] = curr_module
+
+                elif line.startswith("publish"):
+                    curr_publishers = [w for w in line.replace('\n', '').replace(' ', '').split('=')[1].split(',')]
+                    modules[curr_module]['pub'] = curr_publishers
+                    for pub in curr_publishers:
+                        streamingPub[pub] = curr_module
+                else:
+                    continue
+
+    output_set_graph = set()
+
+    for module in modules.keys():
+        for stream_in in modules[module]['sub']:
+            if stream_in not in streamingPub.keys():
+                output_set_graph.add("\"" + stream_in + "\" [color=darkorange1] ;\n")
+                output_set_graph.add("\"" + stream_in + "\"" + "->" + module + ";\n")
+            else:
+                output_set_graph.add("\"" + streamingPub[stream_in] + "\"" + "->" + module + ";\n")
+
+        for stream_out in modules[module]['pub']:
+            if stream_out not in streamingSub.keys():
+                output_set_graph.add("\"" + stream_out + "\" [color=darkorange1] ;\n")
+                output_set_graph.add("\"" + stream_out + "\"" + "->" + module + ";\n")
+            else:
+                output_set_graph.add("\"" + module + "\"" + "->" + streamingSub[stream_out] + ";\n")
+
+
+    output_text_graph = ""
+    output_text_graph += "digraph unix {\n"\
+                              "graph [pad=\"0.5\"];\n"\
+                              "size=\"25,25\";\n"\
+                              "node [color=lightblue2, style=filled];\n"
+
+    for elem in output_set_graph:
+        output_text_graph += elem
+
+    output_text_graph += "}"
+    print output_text_graph

doc/generate_modules_data_flow_graph.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python generate_graph_data.py | dot -T png -o module-data-flow.png

installing_deps.sh

@@ -17,6 +17,9 @@ sudo apt-get install libadns1 libadns1-dev
 #Needed for redis-lvlDB
 sudo apt-get install libev-dev libgmp-dev
 
+#Need for generate-data-flow graph
+sudo apt-get install graphviz
+
 #needed for mathplotlib
 test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
 sudo easy_install -U distribute