Revert "Added more documentation/descriptions/..."

pull/248/head
Thirion Aurélien 2018-09-11 13:51:57 +02:00 committed by GitHub
parent 544561acd5
commit fcefb21da0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 70 additions and 40 deletions

View File

@ -8,6 +8,11 @@ It trying to extract Bitcoin address and secret key from paste
..seealso:: Paste method (get_regex)
Requirements
------------
*Need running Redis instances. (Redis).
"""
from packages import Paste

View File

@ -4,9 +4,12 @@
The ZMQ_PubSub_Categ Module
============================
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
Module.
Each words files created under /files/ are representing categories.
This modules take these files and compare them to
the stream of data.
the stream of data given by the ZMQ_PubSub_Tokenize_Q Module.
When a word from a paste match one or more of these words file, the filename of
the paste is published/forwarded to the next modules.
@ -22,10 +25,15 @@ Implementing modules can start here, create your own category file,
and then create your own module to treat the specific paste matching this
category.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (Redis)
*Categories files of words in /files/ need to be created
*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
"""
import os

View File

@ -162,13 +162,3 @@ if __name__ == "__main__":
for partCred in splitedCred:
if len(partCred) > minimumLengthThreshold:
server_cred.sadd(partCred, uniq_num_cred)
build_top_username(uniq_num_path, paste._get_p_date(), server_cred)
# hset:
# mostPostedUser:DATE -> uniq_cred_id -> count
def build_top_username(uniq, date, serv):
pass

View File

@ -1,11 +1,16 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
This modules update a .csv file used to draw curves representing selected
words and their occurency per day.
..note:: The channel will have the name of the file created.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
This Module is also used for term frequency.

View File

@ -3,7 +3,7 @@
"""
Decoder module
Dectect and decode content based on the provided decoder function.
Dectect Binary and decode it
"""
import time
import os

View File

@ -6,7 +6,7 @@ The DomClassifier Module
============================
The DomClassifier modules extract and classify Internet domains/hostnames/IP addresses from
the output of the Global module. Also performs DNS lookup.
the out output of the Global module.
"""
import time

View File

@ -9,6 +9,8 @@ This huge module is, in short term, checking duplicates.
Its input comes from other modules, namely:
Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone
This one differ from v1 by only using redis and not json file stored on disk
Perform comparisions with ssdeep and tlsh
Requirements:

View File

@ -4,12 +4,21 @@
The ZMQ_Feed_Q Module
=====================
This module is consuming the Redis-list created by the Mixer Module,
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module,
And save the paste on disk to allow others modules to work on them.
..todo:: Be able to choose to delete or not the saved paste after processing.
..todo:: Store the empty paste (unprocessed) somewhere in Redis.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances.
*Need the ZMQ_Feed_Q Module running to be able to work properly.
"""
import base64
import os

View File

@ -7,6 +7,9 @@ Queue helper module
This module subscribe to a Publisher stream and put the received messages
into a Redis-list waiting to be popped later by others scripts.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
"""
import redis
import configparser

View File

@ -2,10 +2,10 @@
# -*-coding:UTF-8 -*
"""
The Indexer Module
The ZMQ_Sub_Indexer Module
============================
This module is fetching the list of files to be processed
The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
and index each file with a full-text indexer (Whoosh until now).
"""

View File

@ -2,10 +2,11 @@
# -*-coding:UTF-8 -*
"""
The Lines Module
The ZMQ_PubSub_Lines Module
============================
This module is consuming the Redis-list created by Global Module.
This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q
Module.
It perform a sorting on the line's length and publish/forward them to
differents channels:
@ -16,6 +17,15 @@ differents channels:
The collected informations about the processed pastes
(number of lines and maximum length line) are stored in Redis.
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (LevelDB & Redis)
*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly.
"""
import argparse
import time

View File

@ -2,7 +2,7 @@
# -*-coding:UTF-8 -*
"""
The Mail Module
The CreditCards Module
======================
This module is consuming the Redis-list created by the Categ module.

View File

@ -4,8 +4,7 @@
The Mixer Module
================
This module is consuming the Redis-list created by the ZMQ_Feed Module.
It is responsible to feeding data to the all plateform.
This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
This module take all the feeds provided in the config.
Depending on the configuration, this module will process the feed as follow:

View File

@ -1,9 +1,5 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
This module can be used to see the health of modules in the system, and also
perform basics statistics about performances.
'''
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
Button, Label

View File

@ -1,14 +1,25 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Onion Module
The ZMQ_Sub_Onion Module
============================
This module is consuming the Redis-list created by the ZMQ_Sub_Onion_Q Module.
It trying to extract url from paste and returning only ones which are tor
related (.onion)
..seealso:: Paste method (get_regex)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------
*Need running Redis instances. (Redis)
*Need the ZMQ_Sub_Onion_Q Module running to be able to work properly.
"""
import pprint
import time

View File

@ -5,7 +5,7 @@
The Tags Module
================================
This module create tags on pastes.
This module create tags.
"""
import redis

View File

@ -4,7 +4,8 @@
The Tokenize Module
===================
This module is consuming the Redis-list created by the Global Module.
This module is consuming the Redis-list created by the ZMQ_PubSub_Tokenize_Q
Module.
It tokenize the content of the paste and publish the result in the following
format:
@ -12,6 +13,9 @@ format:
..seealso:: Paste method (_get_top_words)
..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
the same Subscriber name in both of them.
Requirements
------------

View File

@ -8,10 +8,6 @@ from collections import OrderedDict
import sys
import shutil
'''
Helper module to update the configuration file, looking if the config is up-to-date.
'''
#return true if the configuration is up-to-date
def main():

View File

@ -1,10 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
This module can be used to import a directory and its subsequent folder in AIL.
''
import zmq
import base64
from io import StringIO

View File

@ -1,10 +1,6 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
This module can be used to manually submit a paste to the system.
'''
import configparser
import os
import sys