diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py index 7958787d..5ec2199f 100755 --- a/bin/Bitcoin.py +++ b/bin/Bitcoin.py @@ -8,6 +8,11 @@ It trying to extract Bitcoin address and secret key from paste ..seealso:: Paste method (get_regex) +Requirements +------------ + +*Need running Redis instances. (Redis). + """ from packages import Paste diff --git a/bin/Categ.py b/bin/Categ.py index 175c3d76..cf78f90f 100755 --- a/bin/Categ.py +++ b/bin/Categ.py @@ -4,9 +4,12 @@ The ZMQ_PubSub_Categ Module ============================ +This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q +Module. + Each words files created under /files/ are representing categories. This modules take these files and compare them to -the stream of data. +the stream of data given by the ZMQ_PubSub_Tokenize_Q Module. When a word from a paste match one or more of these words file, the filename of the paste is published/forwarded to the next modules. @@ -22,10 +25,15 @@ Implementing modules can start here, create your own category file, and then create your own module to treat the specific paste matching this category. +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + Requirements ------------ +*Need running Redis instances. (Redis) *Categories files of words in /files/ need to be created +*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly. """ import os diff --git a/bin/Credential.py b/bin/Credential.py index bddb88ce..d1016586 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -162,13 +162,3 @@ if __name__ == "__main__": for partCred in splitedCred: if len(partCred) > minimumLengthThreshold: server_cred.sadd(partCred, uniq_num_cred) - - build_top_username(uniq_num_path, paste._get_p_date(), server_cred) - -# hset: -# mostPostedUser:DATE -> uniq_cred_id -> count -def build_top_username(uniq, date, serv): - pass - - - diff --git a/bin/Curve.py b/bin/Curve.py index 045128c8..8e228039 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -1,11 +1,16 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* """ +This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module. + This modules update a .csv file used to draw curves representing selected words and their occurency per day. ..note:: The channel will have the name of the file created. +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + This Module is also used for term frequency. diff --git a/bin/Decoder.py b/bin/Decoder.py index 534ede3b..af385fed 100755 --- a/bin/Decoder.py +++ b/bin/Decoder.py @@ -3,7 +3,7 @@ """ Decoder module - Dectect and decode content based on the provided decoder function. + Dectect Binary and decode it """ import time import os diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py index 37d041ea..aed87a55 100755 --- a/bin/DomClassifier.py +++ b/bin/DomClassifier.py @@ -6,7 +6,7 @@ The DomClassifier Module ============================ The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from -the output of the Global module. Also performs DNS lookup. +the out output of the Global module. """ import time diff --git a/bin/Duplicates.py b/bin/Duplicates.py index 4d8fffa4..0c24bec1 100755 --- a/bin/Duplicates.py +++ b/bin/Duplicates.py @@ -9,6 +9,8 @@ This huge module is, in short term, checking duplicates. Its input comes from other modules, namely: Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone +This one differ from v1 by only using redis and not json file stored on disk + Perform comparisions with ssdeep and tlsh Requirements: diff --git a/bin/Global.py b/bin/Global.py index a4863ab0..32a3656b 100755 --- a/bin/Global.py +++ b/bin/Global.py @@ -4,12 +4,21 @@ The ZMQ_Feed_Q Module ===================== -This module is consuming the Redis-list created by the Mixer Module, +This module is consuming the Redis-list created by the ZMQ_Feed_Q Module, And save the paste on disk to allow others modules to work on them. ..todo:: Be able to choose to delete or not the saved paste after processing. ..todo:: Store the empty paste (unprocessed) somewhere in Redis. +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + +Requirements +------------ + +*Need running Redis instances. +*Need the ZMQ_Feed_Q Module running to be able to work properly. + """ import base64 import os diff --git a/bin/Helper.py b/bin/Helper.py index 4f6f3d7a..d90388f5 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -7,6 +7,9 @@ Queue helper module This module subscribe to a Publisher stream and put the received messages into a Redis-list waiting to be popped later by others scripts. +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + """ import redis import configparser diff --git a/bin/Indexer.py b/bin/Indexer.py index 87eb9344..1d1ece4b 100755 --- a/bin/Indexer.py +++ b/bin/Indexer.py @@ -2,10 +2,10 @@ # -*-coding:UTF-8 -* """ -The Indexer Module +The ZMQ_Sub_Indexer Module ============================ -This module is fetching the list of files to be processed +The ZMQ_Sub_Indexer modules is fetching the list of files to be processed and index each file with a full-text indexer (Whoosh until now). """ diff --git a/bin/Lines.py b/bin/Lines.py index f357a0b2..8c9f6827 100755 --- a/bin/Lines.py +++ b/bin/Lines.py @@ -2,10 +2,11 @@ # -*-coding:UTF-8 -* """ -The Lines Module +The ZMQ_PubSub_Lines Module ============================ -This module is consuming the Redis-list created by Global Module. +This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q +Module. It perform a sorting on the line's length and publish/forward them to differents channels: @@ -16,6 +17,15 @@ differents channels: The collected informations about the processed pastes (number of lines and maximum length line) are stored in Redis. +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + +Requirements +------------ + +*Need running Redis instances. (LevelDB & Redis) +*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly. + """ import argparse import time diff --git a/bin/Mail.py b/bin/Mail.py index 139e8245..c1d8cf70 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -2,7 +2,7 @@ # -*-coding:UTF-8 -* """ -The Mail Module +The CreditCards Module ====================== This module is consuming the Redis-list created by the Categ module. diff --git a/bin/Mixer.py b/bin/Mixer.py index 7423e99b..96f20815 100755 --- a/bin/Mixer.py +++ b/bin/Mixer.py @@ -4,8 +4,7 @@ The Mixer Module ================ -This module is consuming the Redis-list created by the ZMQ_Feed Module. -It is responsible to feeding data to the all plateform. +This module is consuming the Redis-list created by the ZMQ_Feed_Q Module. This module take all the feeds provided in the config. Depending on the configuration, this module will process the feed as follow: diff --git a/bin/ModulesInformationV2.py b/bin/ModulesInformationV2.py index cfdd8548..36b397ca 100755 --- a/bin/ModulesInformationV2.py +++ b/bin/ModulesInformationV2.py @@ -1,9 +1,5 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' -This module can be used to see the health of modules in the system, and also -perform basics statistics about performances. -''' from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \ Button, Label diff --git a/bin/Onion.py b/bin/Onion.py index f1435b10..277f1c71 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -1,14 +1,25 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* """ -The Onion Module +The ZMQ_Sub_Onion Module ============================ +This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module. + It trying to extract url from paste and returning only ones which are tor related (.onion) ..seealso:: Paste method (get_regex) +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + +Requirements +------------ + +*Need running Redis instances. (Redis) +*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly. + """ import pprint import time diff --git a/bin/Tags.py b/bin/Tags.py index d16da6fa..0a178fef 100755 --- a/bin/Tags.py +++ b/bin/Tags.py @@ -5,7 +5,7 @@ The Tags Module ================================ -This module create tags on pastes. +This module create tags. """ import redis diff --git a/bin/Tokenize.py b/bin/Tokenize.py index 8b6f2812..698b4fbc 100755 --- a/bin/Tokenize.py +++ b/bin/Tokenize.py @@ -4,7 +4,8 @@ The Tokenize Module =================== -This module is consuming the Redis-list created by the Global Module. +This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q +Module. It tokenize the content of the paste and publish the result in the following format: @@ -12,6 +13,9 @@ format: ..seealso:: Paste method (_get_top_words) +..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put +the same Subscriber name in both of them. + Requirements ------------ diff --git a/bin/Update-conf.py b/bin/Update-conf.py index 4ca80837..0d04fb88 100755 --- a/bin/Update-conf.py +++ b/bin/Update-conf.py @@ -8,10 +8,6 @@ from collections import OrderedDict import sys import shutil -''' -Helper module to update the configuration file, looking if the config is up-to-date. -''' - #return true if the configuration is up-to-date def main(): diff --git a/bin/import_dir.py b/bin/import_dir.py index c91bcd96..d8360631 100755 --- a/bin/import_dir.py +++ b/bin/import_dir.py @@ -1,10 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -''' -This module can be used to import a directory and its subsequent folder in AIL. -'' - import zmq import base64 from io import StringIO diff --git a/bin/submit_paste.py b/bin/submit_paste.py index 3193db70..a999ec39 100755 --- a/bin/submit_paste.py +++ b/bin/submit_paste.py @@ -1,10 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' -This module can be used to manually submit a paste to the system. -''' - import configparser import os import sys