mirror of https://github.com/CIRCL/AIL-framework
36 lines
866 B
Python
36 lines
866 B
Python
|
#!/usr/bin/env python2
|
||
|
|
# -*-coding:UTF-8 -*
|
||
|
|
|
||
|
|
"""
|
||
|
|
The Duplicate module
|
||
|
|
====================
|
||
|
|
|
||
|
|
This huge module is, in short term, checking duplicates.
|
||
|
|
|
||
|
|
Requirements:
|
||
|
|
-------------
|
||
|
|
|
||
|
|
|
||
|
|
"""
|
||
|
|
import redis
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
from packages import Paste
|
||
|
|
from pubsublogger import publisher
|
||
|
|
from Helper import Process
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
publisher.port = 6380
|
||
|
|
publisher.channel = "Script"
|
||
|
|
|
||
|
|
config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
|
||
|
|
|
||
|
|
for queue in config_section:
|
||
|
|
print 'dropping: ' + queue
|
||
|
|
p = Process(queue)
|
||
|
|
while True:
|
||
|
|
message = p.get_from_set()
|
||
|
|
if message is None:
|
||
|
|
break
|
||
|
|
|