From fcefb21da013fd904f3b89c5cab40c6446a7016d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= <Terrtia@users.noreply.github.com>
Date: Tue, 11 Sep 2018 13:51:57 +0200
Subject: [PATCH] Revert "Added more documentation/descriptions/..."

---
 bin/Bitcoin.py              |  5 +++++
 bin/Categ.py                | 10 +++++++++-
 bin/Credential.py           | 10 ----------
 bin/Curve.py                |  5 +++++
 bin/Decoder.py              |  2 +-
 bin/DomClassifier.py        |  2 +-
 bin/Duplicates.py           |  2 ++
 bin/Global.py               | 11 ++++++++++-
 bin/Helper.py               |  3 +++
 bin/Indexer.py              |  4 ++--
 bin/Lines.py                | 14 ++++++++++++--
 bin/Mail.py                 |  2 +-
 bin/Mixer.py                |  3 +--
 bin/ModulesInformationV2.py |  4 ----
 bin/Onion.py                | 13 ++++++++++++-
 bin/Tags.py                 |  2 +-
 bin/Tokenize.py             |  6 +++++-
 bin/Update-conf.py          |  4 ----
 bin/import_dir.py           |  4 ----
 bin/submit_paste.py         |  4 ----
 20 files changed, 70 insertions(+), 40 deletions(-)

diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py
index 7958787d..5ec2199f 100755
--- a/bin/Bitcoin.py
+++ b/bin/Bitcoin.py
@@ -8,6 +8,11 @@ It trying to extract Bitcoin address and secret key from paste
 
     ..seealso:: Paste method (get_regex)
 
+Requirements
+------------
+
+*Need running Redis instances. (Redis).
+
 """
 
 from packages import Paste
diff --git a/bin/Categ.py b/bin/Categ.py
index 175c3d76..cf78f90f 100755
--- a/bin/Categ.py
+++ b/bin/Categ.py
@@ -4,9 +4,12 @@
 The ZMQ_PubSub_Categ Module
 ============================
 
+This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
+Module.
+
 Each words files created under /files/ are representing categories.
 This modules take these files and compare them to
-the stream of data.
+the stream of data given by the ZMQ_PubSub_Tokenize_Q  Module.
 
 When a word from a paste match one or more of these words file, the filename of
 the paste is published/forwarded to the next modules.
@@ -22,10 +25,15 @@ Implementing modules can start here, create your own category file,
 and then create your own module to treat the specific paste matching this
 category.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 Requirements
 ------------
 
+*Need running Redis instances. (Redis)
 *Categories files of words in /files/ need to be created
+*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
 
 """
 import os
diff --git a/bin/Credential.py b/bin/Credential.py
index bddb88ce..d1016586 100755
--- a/bin/Credential.py
+++ b/bin/Credential.py
@@ -162,13 +162,3 @@ if __name__ == "__main__":
             for partCred in splitedCred:
                 if len(partCred) > minimumLengthThreshold:
                     server_cred.sadd(partCred, uniq_num_cred)
-
-            build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
-
-# hset:
-# mostPostedUser:DATE -> uniq_cred_id -> count
-def build_top_username(uniq, date, serv):
-    pass
-
-
-
diff --git a/bin/Curve.py b/bin/Curve.py
index 045128c8..8e228039 100755
--- a/bin/Curve.py
+++ b/bin/Curve.py
@@ -1,11 +1,16 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
+This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
+
 This modules update a .csv file used to draw curves representing selected
 words and their occurency per day.
 
 ..note:: The channel will have the name of the file created.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 
 This Module is also used for term frequency.
 
diff --git a/bin/Decoder.py b/bin/Decoder.py
index 534ede3b..af385fed 100755
--- a/bin/Decoder.py
+++ b/bin/Decoder.py
@@ -3,7 +3,7 @@
 """
     Decoder module
 
-    Dectect and decode content based on the provided decoder function.
+    Dectect Binary and decode it
 """
 import time
 import os
diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py
index 37d041ea..aed87a55 100755
--- a/bin/DomClassifier.py
+++ b/bin/DomClassifier.py
@@ -6,7 +6,7 @@ The DomClassifier Module
 ============================
 
 The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
-the output of the Global module. Also performs DNS lookup.
+the out output of the Global module.
 
 """
 import time
diff --git a/bin/Duplicates.py b/bin/Duplicates.py
index 4d8fffa4..0c24bec1 100755
--- a/bin/Duplicates.py
+++ b/bin/Duplicates.py
@@ -9,6 +9,8 @@ This huge module is, in short term, checking duplicates.
 Its input comes from other modules, namely:
     Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
 
+This one differ from v1 by only using redis and not json file stored on disk
+
 Perform comparisions with ssdeep and tlsh
 
 Requirements:
diff --git a/bin/Global.py b/bin/Global.py
index a4863ab0..32a3656b 100755
--- a/bin/Global.py
+++ b/bin/Global.py
@@ -4,12 +4,21 @@
 The ZMQ_Feed_Q Module
 =====================
 
-This module is consuming the Redis-list created by the Mixer Module,
+This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
 And save the paste on disk to allow others modules to work on them.
 
 ..todo:: Be able to choose to delete or not the saved paste after processing.
 ..todo:: Store the empty paste (unprocessed) somewhere in Redis.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
+Requirements
+------------
+
+*Need running Redis instances.
+*Need the ZMQ_Feed_Q Module running to be able to work properly.
+
 """
 import base64
 import os
diff --git a/bin/Helper.py b/bin/Helper.py
index 4f6f3d7a..d90388f5 100755
--- a/bin/Helper.py
+++ b/bin/Helper.py
@@ -7,6 +7,9 @@ Queue helper module
 This module subscribe to a Publisher stream and put the received messages
 into a Redis-list waiting to be popped later by others scripts.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 """
 import redis
 import configparser
diff --git a/bin/Indexer.py b/bin/Indexer.py
index 87eb9344..1d1ece4b 100755
--- a/bin/Indexer.py
+++ b/bin/Indexer.py
@@ -2,10 +2,10 @@
 # -*-coding:UTF-8 -*
 
 """
-The Indexer Module
+The ZMQ_Sub_Indexer Module
 ============================
 
-This module is fetching the list of files to be processed
+The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
 and index each file with a full-text indexer (Whoosh until now).
 
 """
diff --git a/bin/Lines.py b/bin/Lines.py
index f357a0b2..8c9f6827 100755
--- a/bin/Lines.py
+++ b/bin/Lines.py
@@ -2,10 +2,11 @@
 # -*-coding:UTF-8 -*
 
 """
-The Lines Module
+The ZMQ_PubSub_Lines Module
 ============================
 
-This module is consuming the Redis-list created by Global Module.
+This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
+Module.
 
 It perform a sorting on the line's length and publish/forward them to
 differents channels:
@@ -16,6 +17,15 @@ differents channels:
 The collected informations about the processed pastes
 (number of lines and maximum length line) are stored in Redis.
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
+Requirements
+------------
+
+*Need running Redis instances. (LevelDB & Redis)
+*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
+
 """
 import argparse
 import time
diff --git a/bin/Mail.py b/bin/Mail.py
index 139e8245..c1d8cf70 100755
--- a/bin/Mail.py
+++ b/bin/Mail.py
@@ -2,7 +2,7 @@
 # -*-coding:UTF-8 -*
 
 """
-The Mail Module
+The CreditCards Module
 ======================
 
 This module is consuming the Redis-list created by the Categ module.
diff --git a/bin/Mixer.py b/bin/Mixer.py
index 7423e99b..96f20815 100755
--- a/bin/Mixer.py
+++ b/bin/Mixer.py
@@ -4,8 +4,7 @@
 The Mixer Module
 ================
 
-This module is consuming the Redis-list created by the ZMQ_Feed Module.
-It is responsible to feeding data to the all plateform.
+This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
 
 This module take all the feeds provided in the config.
 Depending on the configuration, this module will process the feed as follow:
diff --git a/bin/ModulesInformationV2.py b/bin/ModulesInformationV2.py
index cfdd8548..36b397ca 100755
--- a/bin/ModulesInformationV2.py
+++ b/bin/ModulesInformationV2.py
@@ -1,9 +1,5 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
-'''
-This module can be used to see the health of modules in the system, and also
-perform basics statistics about performances.
-'''
 
 from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
     Button, Label
diff --git a/bin/Onion.py b/bin/Onion.py
index f1435b10..277f1c71 100755
--- a/bin/Onion.py
+++ b/bin/Onion.py
@@ -1,14 +1,25 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-The Onion Module
+The ZMQ_Sub_Onion Module
 ============================
 
+This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
+
 It trying to extract url from paste and returning only ones which are tor
 related (.onion)
 
     ..seealso:: Paste method (get_regex)
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
+Requirements
+------------
+
+*Need running Redis instances. (Redis)
+*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
+
 """
 import pprint
 import time
diff --git a/bin/Tags.py b/bin/Tags.py
index d16da6fa..0a178fef 100755
--- a/bin/Tags.py
+++ b/bin/Tags.py
@@ -5,7 +5,7 @@
 The Tags Module
 ================================
 
-This module create tags on pastes.
+This module create tags.
 
 """
 import redis
diff --git a/bin/Tokenize.py b/bin/Tokenize.py
index 8b6f2812..698b4fbc 100755
--- a/bin/Tokenize.py
+++ b/bin/Tokenize.py
@@ -4,7 +4,8 @@
 The Tokenize Module
 ===================
 
-This module is consuming the Redis-list created by the Global Module.
+This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
+Module.
 
 It tokenize the content of the paste and publish the result in the following
 format:
@@ -12,6 +13,9 @@ format:
 
     ..seealso:: Paste method (_get_top_words)
 
+..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
+the same Subscriber name in both of them.
+
 Requirements
 ------------
 
diff --git a/bin/Update-conf.py b/bin/Update-conf.py
index 4ca80837..0d04fb88 100755
--- a/bin/Update-conf.py
+++ b/bin/Update-conf.py
@@ -8,10 +8,6 @@ from collections import OrderedDict
 import sys
 import shutil
 
-'''
-Helper module to update the configuration file, looking if the config is up-to-date.
-'''
-
 
 #return true if the configuration is up-to-date
 def main():
diff --git a/bin/import_dir.py b/bin/import_dir.py
index c91bcd96..d8360631 100755
--- a/bin/import_dir.py
+++ b/bin/import_dir.py
@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-'''
-This module can be used to import a directory and its subsequent folder in AIL.
-''
-
 import zmq
 import base64
 from io import StringIO
diff --git a/bin/submit_paste.py b/bin/submit_paste.py
index 3193db70..a999ec39 100755
--- a/bin/submit_paste.py
+++ b/bin/submit_paste.py
@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-'''
-This module can be used to manually submit a paste to the system.
-'''
-
 import configparser
 import os
 import sys