From 12aca6b760231dca8c289cca30bdc97c2f958a23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 4 Feb 2016 15:22:51 +0100
Subject: [PATCH] Add script to import from local directory, use local python
 from env

---
 bin/Attribute.py     |  2 +-
 bin/Categ.py         |  2 +-
 bin/CreditCard.py    |  2 +-
 bin/DomClassifier.py |  2 +-
 bin/Global.py        |  2 +-
 bin/Indexer.py       |  2 +-
 bin/Line.py          |  2 +-
 bin/Mail.py          |  2 +-
 bin/Onion.py         |  2 +-
 bin/Tokenize.py      |  2 +-
 bin/Url.py           |  2 +-
 bin/import_dir.py    | 26 ++++++++++++++++++++++++++
 bin/launch_queues.py |  2 +-
 13 files changed, 38 insertions(+), 12 deletions(-)
 create mode 100755 bin/import_dir.py

diff --git a/bin/Attribute.py b/bin/Attribute.py
index 4235bb4e..46d80858 100755
--- a/bin/Attribute.py
+++ b/bin/Attribute.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 
 """
diff --git a/bin/Categ.py b/bin/Categ.py
index 8bf834a1..15af4f52 100755
--- a/bin/Categ.py
+++ b/bin/Categ.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 """
 The ZMQ_PubSub_Categ Module
diff --git a/bin/CreditCard.py b/bin/CreditCard.py
index b1bdc26e..d4660c13 100755
--- a/bin/CreditCard.py
+++ b/bin/CreditCard.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 import pprint
 import time
diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py
index f1f59f72..0712e959 100755
--- a/bin/DomClassifier.py
+++ b/bin/DomClassifier.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 
 """
diff --git a/bin/Global.py b/bin/Global.py
index e3181580..fb44c70b 100755
--- a/bin/Global.py
+++ b/bin/Global.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 """
 The ZMQ_Feed_Q Module
diff --git a/bin/Indexer.py b/bin/Indexer.py
index 679ac8e0..39f0fe51 100755
--- a/bin/Indexer.py
+++ b/bin/Indexer.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 
 """
diff --git a/bin/Line.py b/bin/Line.py
index 81dc9483..be1ad635 100755
--- a/bin/Line.py
+++ b/bin/Line.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 
 """
diff --git a/bin/Mail.py b/bin/Mail.py
index 07d742b2..dd348ba6 100755
--- a/bin/Mail.py
+++ b/bin/Mail.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 
 import redis
diff --git a/bin/Onion.py b/bin/Onion.py
index c47174bc..cdfb44bc 100755
--- a/bin/Onion.py
+++ b/bin/Onion.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 """
 The ZMQ_Sub_Onion Module
diff --git a/bin/Tokenize.py b/bin/Tokenize.py
index b1a75495..b0adf895 100755
--- a/bin/Tokenize.py
+++ b/bin/Tokenize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 """
 The ZMQ_PubSub_Lines Module
diff --git a/bin/Url.py b/bin/Url.py
index e8061895..472d73b3 100755
--- a/bin/Url.py
+++ b/bin/Url.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*
 import redis
 import pprint
diff --git a/bin/import_dir.py b/bin/import_dir.py
new file mode 100755
index 00000000..dd3777b0
--- /dev/null
+++ b/bin/import_dir.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import zmq
+import base64
+import argparse
+import os
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Take files from a directory and push them into a 0MQ feed.')
+    parser.add_argument('-d', '--directory', type=str, required=True, help='Root directory to import')
+    parser.add_argument('-p', '--port', type=int, default=5556, help='Zero MQ port')
+    parser.add_argument('-c', '--channel', type=str, default='102', help='Zero MQ channel')
+
+    args = parser.parse_args()
+
+    context = zmq.Context()
+    socket = context.socket(zmq.PUB)
+    socket.bind("tcp://*:{}".format(args.port))
+
+    for dirname, dirnames, filenames in os.walk(args.directory):
+        for filename in filenames:
+            messagedata = open(os.path.join(dirname, filename)).read()
+            print(os.path.join(dirname, filename))
+            socket.send('{} {} {}'.format(args.channel, filename, base64.b64encode(messagedata)))
diff --git a/bin/launch_queues.py b/bin/launch_queues.py
index 05adf774..e60a7b50 100755
--- a/bin/launch_queues.py
+++ b/bin/launch_queues.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*-coding:UTF-8 -*