Cleanup.

* Remove useless subscriber
* Fix typo in the config file
* Update Helper accordingly

bin/Helper.py

@@ -31,6 +31,7 @@ class Redis_Queues(object):
         self.subscriber_name = subscriber_name
 
         self.sub_channel = self.config.get(conf_section, conf_channel)
+        self.sub_address = self.config.get(conf_section, 'address')
         self.redis_channel = self.sub_channel + self.subscriber_name
 
         # Redis Queue
@@ -40,11 +41,10 @@ class Redis_Queues(object):
             port=self.config.getint(config_section, "port"),
             db=self.config.getint(config_section, "db"))
 
-    def zmq_sub(self, conf_section):
-        sub_address = self.config.get(conf_section, 'adress')
+    def zmq_sub(self):
         context = zmq.Context()
         self.sub_socket = context.socket(zmq.SUB)
-        self.sub_socket.connect(sub_address)
+        self.sub_socket.connect(self.sub_address)
         self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.sub_channel)
 
     def zmq_pub(self, config_section, config_channel):
@@ -74,6 +74,7 @@ class Redis_Queues(object):
         return self.r_queues.srem('SHUTDOWN_FLAGS', flag)
 
     def redis_queue_subscribe(self, publisher):
+        self.zmq_sub()
         publisher.info("Suscribed to channel {}".format(self.sub_channel))
         while True:
             msg = self.sub_socket.recv()

bin/ZMQ_Feed_Q.py

@@ -34,5 +34,4 @@ if __name__ == "__main__":
     subscriber_name = 'feed'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_PubSub_Categ_Q.py

@@ -32,5 +32,4 @@ if __name__ == "__main__":
     subscriber_name = 'categ'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_PubSub_Lines.py

@@ -48,9 +48,6 @@ if __name__ == "__main__":
     pub_config_section = 'PubSub_Longlines'
     h.zmq_pub(pub_config_section, None)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # SCRIPT PARSER #
     parser = argparse.ArgumentParser(
         description='''This script is a part of the Analysis Information \

bin/ZMQ_PubSub_Lines_Q.py

@@ -31,5 +31,4 @@ if __name__ == "__main__":
     subscriber_name = 'line'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_PubSub_Tokenize_Q.py

@@ -32,5 +32,4 @@ if __name__ == "__main__":
     subscriber_name = 'tokenize'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Attributes.py

@@ -42,9 +42,6 @@ if __name__ == "__main__":
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # FUNCTIONS #
     publisher.info("""ZMQ Attribute is Running""")
 

bin/ZMQ_Sub_Attributes_Q.py

@@ -32,5 +32,4 @@ if __name__ == "__main__":
     subscriber_name = 'attributes'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_CreditCards.py

@@ -18,9 +18,6 @@ if __name__ == "__main__":
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # FUNCTIONS #
     publisher.info("Creditcard script subscribed to channel creditcard_categ")
 

bin/ZMQ_Sub_CreditCards_Q.py

@@ -15,5 +15,4 @@ if __name__ == "__main__":
     subscriber_name = 'cards'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Curve.py

@@ -41,9 +41,6 @@ if __name__ == "__main__":
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # REDIS #
     r_serv1 = redis.StrictRedis(
         host=h.config.get("Redis_Level_DB", "host"),

bin/ZMQ_Sub_Curve_Q.py

@@ -32,5 +32,4 @@ if __name__ == "__main__":
     subscriber_name = 'curve'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Duplicate.py

@@ -31,9 +31,6 @@ if __name__ == "__main__":
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # REDIS #
     # DB OBJECT & HASHS ( DISK )
     # FIXME increase flexibility

bin/ZMQ_Sub_Duplicate_Q.py

@@ -14,5 +14,4 @@ if __name__ == "__main__":
     subscriber_name = 'duplicate'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Indexer_Q.py

@@ -26,5 +26,4 @@ if __name__ == "__main__":
     subscriber_name = 'indexer'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Mails.py

@@ -21,9 +21,6 @@ if __name__ == "__main__":
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # REDIS #
     r_serv2 = redis.StrictRedis(
         host=h.config.get("Redis_Cache", "host"),

bin/ZMQ_Sub_Mails_Q.py

@@ -14,5 +14,4 @@ if __name__ == "__main__":
     subscriber_name = 'emails'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Onion.py

@@ -39,9 +39,6 @@ if __name__ == "__main__":
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # FUNCTIONS #
     publisher.info("Script subscribed to channel onion_categ")
 

bin/ZMQ_Sub_Onion_Q.py

@@ -31,5 +31,4 @@ if __name__ == "__main__":
     subscriber_name = 'tor'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/ZMQ_Sub_Urls.py

@@ -31,9 +31,6 @@ if __name__ == "__main__":
     pub_config_channel = 'channel'
     h.zmq_pub(pub_config_section, pub_config_channel)
 
-    # Subscriber
-    h.zmq_sub(config_section)
-
     # REDIS #
     r_serv2 = redis.StrictRedis(
         host=h.config.get("Redis_Cache", "host"),

bin/ZMQ_Sub_Urls_Q.py

@@ -15,5 +15,4 @@ if __name__ == "__main__":
     subscriber_name = 'urls'
 
     h = Helper.Redis_Queues(config_section, config_channel, subscriber_name)
-    h.zmq_sub(config_section)
     h.redis_queue_subscribe(publisher)

bin/packages/config.cfg.sample

@@ -38,31 +38,31 @@ db = 1
 
 # PUB / SUB : ZMQ
 [Feed]
-adress = tcp://crf.circl.lu:5556
+address = tcp://crf.circl.lu:5556
 topicfilter = 102
 
 [PubSub_Global]
-adress = tcp://127.0.0.1:5000
+address = tcp://127.0.0.1:5000
 channel = filelist
 
 [PubSub_Longlines]
-adress = tcp://127.0.0.1:5001
+address = tcp://127.0.0.1:5001
 channel_0 = Longlines
 channel_1 = Shortlines
 
 [PubSub_Words]
-adress = tcp://127.0.0.1:5002
+address = tcp://127.0.0.1:5002
 channel_0 = words
 
 [PubSub_Categ]
-adress = tcp://127.0.0.1:5003
+address = tcp://127.0.0.1:5003
 channel_0 = creditcard_categ
 channel_1 = mails_categ
 channel_2 = onion_categ
 channel_3 = web_categ
 
 [PubSub_Url]
-adress = tcp://127.0.0.1:5004
+address = tcp://127.0.0.1:5004
 channel = urls
 # country code logged as critical
 cc_critical = DE