Merge pull request #248 from CIRCL/revert-242-moreComments

Revert "Added more documentation/descriptions/..."

bin/Bitcoin.py

@@ -8,6 +8,11 @@ It trying to extract Bitcoin address and secret key from paste
 
     ..seealso:: Paste method (get_regex)
 
+Requirements
+------------
+
+*Need running Redis instances. (Redis).
+
 """
 
 from packages import Paste

bin/Categ.py

@@ -4,9 +4,12 @@
 The ZMQ_PubSub_Categ Module
 ============================
 
+This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
+Module.
+
 Each words files created under /files/ are representing categories.
 This modules take these files and compare them to
-the stream of data.
+the stream of data given by the ZMQ_PubSub_Tokenize_Q  Module.
 
 When a word from a paste match one or more of these words file, the filename of
 the paste is published/forwarded to the next modules.
@@ -22,10 +25,15 @@ Implementing modules can start here, create your own category file,
 and then create your own module to treat the specific paste matching this
 category.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 Requirements
 ------------
 
+*Need running Redis instances. (Redis)
 *Categories files of words in /files/ need to be created
+*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
 
 """
 import os

bin/Credential.py

@@ -162,13 +162,3 @@ if __name__ == "__main__":
             for partCred in splitedCred:
                 if len(partCred) > minimumLengthThreshold:
                     server_cred.sadd(partCred, uniq_num_cred)
-
-            build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
-
-# hset:
-# mostPostedUser:DATE -> uniq_cred_id -> count
-def build_top_username(uniq, date, serv):
-    pass
-
-
-

bin/Curve.py

@@ -1,11 +1,16 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
+This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
+
 This modules update a .csv file used to draw curves representing selected
 words and their occurency per day.
 
 ..note:: The channel will have the name of the file created.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 
 This Module is also used for term frequency.
 

bin/Decoder.py

@@ -3,7 +3,7 @@
 """
     Decoder module
 
-    Dectect and decode content based on the provided decoder function.
+    Dectect Binary and decode it
 """
 import time
 import os

bin/DomClassifier.py

@@ -6,7 +6,7 @@ The DomClassifier Module
 ============================
 
 The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
-the output of the Global module. Also performs DNS lookup.
+the out output of the Global module.
 
 """
 import time

bin/Duplicates.py

@@ -9,6 +9,8 @@ This huge module is, in short term, checking duplicates.
 Its input comes from other modules, namely:
     Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
 
+This one differ from v1 by only using redis and not json file stored on disk
+
 Perform comparisions with ssdeep and tlsh
 
 Requirements:

bin/Global.py

@@ -4,12 +4,21 @@
 The ZMQ_Feed_Q Module
 =====================
 
-This module is consuming the Redis-list created by the Mixer Module,
+This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
 And save the paste on disk to allow others modules to work on them.
 
 ..todo:: Be able to choose to delete or not the saved paste after processing.
 ..todo:: Store the empty paste (unprocessed) somewhere in Redis.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
+Requirements
+------------
+
+*Need running Redis instances.
+*Need the ZMQ_Feed_Q Module running to be able to work properly.
+
 """
 import base64
 import os

bin/Helper.py

@@ -7,6 +7,9 @@ Queue helper module
 This module subscribe to a Publisher stream and put the received messages
 into a Redis-list waiting to be popped later by others scripts.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 """
 import redis
 import configparser

bin/Indexer.py

@@ -2,10 +2,10 @@
 # -*-coding:UTF-8 -*
 
 """
-The Indexer Module
+The ZMQ_Sub_Indexer Module
 ============================
 
-This module is fetching the list of files to be processed
+The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
 and index each file with a full-text indexer (Whoosh until now).
 
 """

bin/Lines.py

@@ -2,10 +2,11 @@
 # -*-coding:UTF-8 -*
 
 """
-The Lines Module
+The ZMQ_PubSub_Lines Module
 ============================
 
-This module is consuming the Redis-list created by Global Module.
+This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
+Module.
 
 It perform a sorting on the line's length and publish/forward them to
 differents channels:
@@ -16,6 +17,15 @@ differents channels:
 The collected informations about the processed pastes
 (number of lines and maximum length line) are stored in Redis.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
+Requirements
+------------
+
+*Need running Redis instances. (LevelDB & Redis)
+*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
+
 """
 import argparse
 import time

bin/Mail.py

@@ -2,7 +2,7 @@
 # -*-coding:UTF-8 -*
 
 """
-The Mail Module
+The CreditCards Module
 ======================
 
 This module is consuming the Redis-list created by the Categ module.

bin/Mixer.py

@@ -4,8 +4,7 @@
 The Mixer Module
 ================
 
-This module is consuming the Redis-list created by the ZMQ_Feed Module.
-It is responsible to feeding data to the all plateform.
+This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
 
 This module take all the feeds provided in the config.
 Depending on the configuration, this module will process the feed as follow:

bin/ModulesInformationV2.py

@@ -1,9 +1,5 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
-'''
-This module can be used to see the health of modules in the system, and also
-perform basics statistics about performances.
-'''
 
 from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
     Button, Label

bin/Onion.py

@@ -1,14 +1,25 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-The Onion Module
+The ZMQ_Sub_Onion Module
 ============================
 
+This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
+
 It trying to extract url from paste and returning only ones which are tor
 related (.onion)
 
     ..seealso:: Paste method (get_regex)
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
+Requirements
+------------
+
+*Need running Redis instances. (Redis)
+*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
+
 """
 import pprint
 import time

bin/Tags.py

@@ -5,7 +5,7 @@
 The Tags Module
 ================================
 
-This module create tags on pastes.
+This module create tags.
 
 """
 import redis

bin/Tokenize.py

@@ -4,7 +4,8 @@
 The Tokenize Module
 ===================
 
-This module is consuming the Redis-list created by the Global Module.
+This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
+Module.
 
 It tokenize the content of the paste and publish the result in the following
 format:
@@ -12,6 +13,9 @@ format:
 
     ..seealso:: Paste method (_get_top_words)
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 Requirements
 ------------
 

bin/Update-conf.py

@@ -8,10 +8,6 @@ from collections import OrderedDict
 import sys
 import shutil
 
-'''
-Helper module to update the configuration file, looking if the config is up-to-date.
-'''
-
 
 #return true if the configuration is up-to-date
 def main():

bin/import_dir.py

@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-'''
-This module can be used to import a directory and its subsequent folder in AIL.
-''
-
 import zmq
 import base64
 from io import StringIO

bin/submit_paste.py

@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-'''
-This module can be used to manually submit a paste to the system.
-'''
-
 import configparser
 import os
 import sys