From 623e876f3b37b31f0d0abf58b18cbc375bab1fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Aug 2014 17:36:57 +0200 Subject: [PATCH] Cleanup. * Remove useless subscriber * Fix typo in the config file * Update Helper accordingly --- bin/Helper.py | 7 ++++--- bin/ZMQ_Feed_Q.py | 1 - bin/ZMQ_PubSub_Categ_Q.py | 1 - bin/ZMQ_PubSub_Lines.py | 3 --- bin/ZMQ_PubSub_Lines_Q.py | 1 - bin/ZMQ_PubSub_Tokenize_Q.py | 1 - bin/ZMQ_Sub_Attributes.py | 3 --- bin/ZMQ_Sub_Attributes_Q.py | 1 - bin/ZMQ_Sub_CreditCards.py | 3 --- bin/ZMQ_Sub_CreditCards_Q.py | 1 - bin/ZMQ_Sub_Curve.py | 3 --- bin/ZMQ_Sub_Curve_Q.py | 1 - bin/ZMQ_Sub_Duplicate.py | 3 --- bin/ZMQ_Sub_Duplicate_Q.py | 1 - bin/ZMQ_Sub_Indexer_Q.py | 1 - bin/ZMQ_Sub_Mails.py | 3 --- bin/ZMQ_Sub_Mails_Q.py | 1 - bin/ZMQ_Sub_Onion.py | 3 --- bin/ZMQ_Sub_Onion_Q.py | 1 - bin/ZMQ_Sub_Urls.py | 3 --- bin/ZMQ_Sub_Urls_Q.py | 1 - bin/packages/config.cfg.sample | 12 ++++++------ 22 files changed, 10 insertions(+), 45 deletions(-) diff --git a/bin/Helper.py b/bin/Helper.py index 92ce56c1..549e3e7d 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -31,6 +31,7 @@ class Redis_Queues(object): self.subscriber_name = subscriber_name self.sub_channel = self.config.get(conf_section, conf_channel) + self.sub_address = self.config.get(conf_section, 'address') self.redis_channel = self.sub_channel + self.subscriber_name # Redis Queue @@ -40,11 +41,10 @@ class Redis_Queues(object): port=self.config.getint(config_section, "port"), db=self.config.getint(config_section, "db")) - def zmq_sub(self, conf_section): - sub_address = self.config.get(conf_section, 'adress') + def zmq_sub(self): context = zmq.Context() self.sub_socket = context.socket(zmq.SUB) - self.sub_socket.connect(sub_address) + self.sub_socket.connect(self.sub_address) self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.sub_channel) def zmq_pub(self, config_section, config_channel): @@ -74,6 +74,7 @@ class Redis_Queues(object): return self.r_queues.srem('SHUTDOWN_FLAGS', flag) def redis_queue_subscribe(self, publisher): + self.zmq_sub() publisher.info("Suscribed to channel {}".format(self.sub_channel)) while True: msg = self.sub_socket.recv() diff --git a/bin/ZMQ_Feed_Q.py b/bin/ZMQ_Feed_Q.py index 9767c059..2bcdd238 100755 --- a/bin/ZMQ_Feed_Q.py +++ b/bin/ZMQ_Feed_Q.py @@ -34,5 +34,4 @@ if __name__ == "__main__": subscriber_name = 'feed' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_PubSub_Categ_Q.py b/bin/ZMQ_PubSub_Categ_Q.py index 08a93e2f..2b836752 100755 --- a/bin/ZMQ_PubSub_Categ_Q.py +++ b/bin/ZMQ_PubSub_Categ_Q.py @@ -32,5 +32,4 @@ if __name__ == "__main__": subscriber_name = 'categ' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_PubSub_Lines.py b/bin/ZMQ_PubSub_Lines.py index abef2e95..f7dd6da2 100755 --- a/bin/ZMQ_PubSub_Lines.py +++ b/bin/ZMQ_PubSub_Lines.py @@ -48,9 +48,6 @@ if __name__ == "__main__": pub_config_section = 'PubSub_Longlines' h.zmq_pub(pub_config_section, None) - # Subscriber - h.zmq_sub(config_section) - # SCRIPT PARSER # parser = argparse.ArgumentParser( description='''This script is a part of the Analysis Information \ diff --git a/bin/ZMQ_PubSub_Lines_Q.py b/bin/ZMQ_PubSub_Lines_Q.py index 3e366d82..6c1d315f 100755 --- a/bin/ZMQ_PubSub_Lines_Q.py +++ b/bin/ZMQ_PubSub_Lines_Q.py @@ -31,5 +31,4 @@ if __name__ == "__main__": subscriber_name = 'line' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_PubSub_Tokenize_Q.py b/bin/ZMQ_PubSub_Tokenize_Q.py index 459ef88e..c3e92dde 100755 --- a/bin/ZMQ_PubSub_Tokenize_Q.py +++ b/bin/ZMQ_PubSub_Tokenize_Q.py @@ -32,5 +32,4 @@ if __name__ == "__main__": subscriber_name = 'tokenize' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Attributes.py b/bin/ZMQ_Sub_Attributes.py index 912f40e6..ce7ddb75 100755 --- a/bin/ZMQ_Sub_Attributes.py +++ b/bin/ZMQ_Sub_Attributes.py @@ -42,9 +42,6 @@ if __name__ == "__main__": h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - # Subscriber - h.zmq_sub(config_section) - # FUNCTIONS # publisher.info("""ZMQ Attribute is Running""") diff --git a/bin/ZMQ_Sub_Attributes_Q.py b/bin/ZMQ_Sub_Attributes_Q.py index 5d1a349f..7047c01e 100755 --- a/bin/ZMQ_Sub_Attributes_Q.py +++ b/bin/ZMQ_Sub_Attributes_Q.py @@ -32,5 +32,4 @@ if __name__ == "__main__": subscriber_name = 'attributes' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_CreditCards.py b/bin/ZMQ_Sub_CreditCards.py index 22a8c5c5..2f21056a 100755 --- a/bin/ZMQ_Sub_CreditCards.py +++ b/bin/ZMQ_Sub_CreditCards.py @@ -18,9 +18,6 @@ if __name__ == "__main__": h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - # Subscriber - h.zmq_sub(config_section) - # FUNCTIONS # publisher.info("Creditcard script subscribed to channel creditcard_categ") diff --git a/bin/ZMQ_Sub_CreditCards_Q.py b/bin/ZMQ_Sub_CreditCards_Q.py index 81a79c7c..80992dad 100755 --- a/bin/ZMQ_Sub_CreditCards_Q.py +++ b/bin/ZMQ_Sub_CreditCards_Q.py @@ -15,5 +15,4 @@ if __name__ == "__main__": subscriber_name = 'cards' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Curve.py b/bin/ZMQ_Sub_Curve.py index c62ae32c..df06e6ea 100755 --- a/bin/ZMQ_Sub_Curve.py +++ b/bin/ZMQ_Sub_Curve.py @@ -41,9 +41,6 @@ if __name__ == "__main__": h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - # Subscriber - h.zmq_sub(config_section) - # REDIS # r_serv1 = redis.StrictRedis( host=h.config.get("Redis_Level_DB", "host"), diff --git a/bin/ZMQ_Sub_Curve_Q.py b/bin/ZMQ_Sub_Curve_Q.py index b7264639..dde959bf 100755 --- a/bin/ZMQ_Sub_Curve_Q.py +++ b/bin/ZMQ_Sub_Curve_Q.py @@ -32,5 +32,4 @@ if __name__ == "__main__": subscriber_name = 'curve' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Duplicate.py b/bin/ZMQ_Sub_Duplicate.py index eca87934..8ebbb8b7 100755 --- a/bin/ZMQ_Sub_Duplicate.py +++ b/bin/ZMQ_Sub_Duplicate.py @@ -31,9 +31,6 @@ if __name__ == "__main__": h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - # Subscriber - h.zmq_sub(config_section) - # REDIS # # DB OBJECT & HASHS ( DISK ) # FIXME increase flexibility diff --git a/bin/ZMQ_Sub_Duplicate_Q.py b/bin/ZMQ_Sub_Duplicate_Q.py index 9fc455ba..e66c3ec4 100755 --- a/bin/ZMQ_Sub_Duplicate_Q.py +++ b/bin/ZMQ_Sub_Duplicate_Q.py @@ -14,5 +14,4 @@ if __name__ == "__main__": subscriber_name = 'duplicate' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Indexer_Q.py b/bin/ZMQ_Sub_Indexer_Q.py index af7e076a..ec5f3418 100755 --- a/bin/ZMQ_Sub_Indexer_Q.py +++ b/bin/ZMQ_Sub_Indexer_Q.py @@ -26,5 +26,4 @@ if __name__ == "__main__": subscriber_name = 'indexer' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Mails.py b/bin/ZMQ_Sub_Mails.py index e3c7e91a..33feef3c 100755 --- a/bin/ZMQ_Sub_Mails.py +++ b/bin/ZMQ_Sub_Mails.py @@ -21,9 +21,6 @@ if __name__ == "__main__": h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - # Subscriber - h.zmq_sub(config_section) - # REDIS # r_serv2 = redis.StrictRedis( host=h.config.get("Redis_Cache", "host"), diff --git a/bin/ZMQ_Sub_Mails_Q.py b/bin/ZMQ_Sub_Mails_Q.py index 7a6e66e9..1ffed251 100755 --- a/bin/ZMQ_Sub_Mails_Q.py +++ b/bin/ZMQ_Sub_Mails_Q.py @@ -14,5 +14,4 @@ if __name__ == "__main__": subscriber_name = 'emails' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Onion.py b/bin/ZMQ_Sub_Onion.py index 2bf27c85..99857520 100755 --- a/bin/ZMQ_Sub_Onion.py +++ b/bin/ZMQ_Sub_Onion.py @@ -39,9 +39,6 @@ if __name__ == "__main__": h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - # Subscriber - h.zmq_sub(config_section) - # FUNCTIONS # publisher.info("Script subscribed to channel onion_categ") diff --git a/bin/ZMQ_Sub_Onion_Q.py b/bin/ZMQ_Sub_Onion_Q.py index 6cbc9938..833edf61 100755 --- a/bin/ZMQ_Sub_Onion_Q.py +++ b/bin/ZMQ_Sub_Onion_Q.py @@ -31,5 +31,4 @@ if __name__ == "__main__": subscriber_name = 'tor' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/ZMQ_Sub_Urls.py b/bin/ZMQ_Sub_Urls.py index 658d9c4a..a339a543 100755 --- a/bin/ZMQ_Sub_Urls.py +++ b/bin/ZMQ_Sub_Urls.py @@ -31,9 +31,6 @@ if __name__ == "__main__": pub_config_channel = 'channel' h.zmq_pub(pub_config_section, pub_config_channel) - # Subscriber - h.zmq_sub(config_section) - # REDIS # r_serv2 = redis.StrictRedis( host=h.config.get("Redis_Cache", "host"), diff --git a/bin/ZMQ_Sub_Urls_Q.py b/bin/ZMQ_Sub_Urls_Q.py index 1e63ae8c..346ca9ec 100755 --- a/bin/ZMQ_Sub_Urls_Q.py +++ b/bin/ZMQ_Sub_Urls_Q.py @@ -15,5 +15,4 @@ if __name__ == "__main__": subscriber_name = 'urls' h = Helper.Redis_Queues(config_section, config_channel, subscriber_name) - h.zmq_sub(config_section) h.redis_queue_subscribe(publisher) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 2483e6c9..b543ee0d 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -38,31 +38,31 @@ db = 1 # PUB / SUB : ZMQ [Feed] -adress = tcp://crf.circl.lu:5556 +address = tcp://crf.circl.lu:5556 topicfilter = 102 [PubSub_Global] -adress = tcp://127.0.0.1:5000 +address = tcp://127.0.0.1:5000 channel = filelist [PubSub_Longlines] -adress = tcp://127.0.0.1:5001 +address = tcp://127.0.0.1:5001 channel_0 = Longlines channel_1 = Shortlines [PubSub_Words] -adress = tcp://127.0.0.1:5002 +address = tcp://127.0.0.1:5002 channel_0 = words [PubSub_Categ] -adress = tcp://127.0.0.1:5003 +address = tcp://127.0.0.1:5003 channel_0 = creditcard_categ channel_1 = mails_categ channel_2 = onion_categ channel_3 = web_categ [PubSub_Url] -adress = tcp://127.0.0.1:5004 +address = tcp://127.0.0.1:5004 channel = urls # country code logged as critical cc_critical = DE