AIL-framework/bin/Indexer.py

73 lines
2.2 KiB
Python
Raw Normal View History

#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
The ZMQ_Sub_Indexer Module
============================
The ZMQ_Sub_Indexer modules is fetching the list of files to be processed
and index each file with a full-text indexer (Whoosh until now).
"""
2014-08-14 17:55:18 +02:00
import time
from packages import Paste
from pubsublogger import publisher
from whoosh.index import create_in, exists_in, open_dir
2014-08-14 17:55:18 +02:00
from whoosh.fields import Schema, TEXT, ID
import os
from Helper import Process
2014-08-14 17:55:18 +02:00
2014-08-19 19:07:07 +02:00
if __name__ == "__main__":
publisher.port = 6380
2014-08-19 19:07:07 +02:00
publisher.channel = "Script"
config_section = 'Indexer'
p = Process(config_section)
# Indexer configuration - index dir and schema setup
indexpath = os.path.join(os.environ['AIL_HOME'],
p.config.get("Indexer", "path"))
indexertype = p.config.get("Indexer", "type")
if indexertype == "whoosh":
2014-08-19 19:07:07 +02:00
schema = Schema(title=TEXT(stored=True), path=ID(stored=True,
unique=True),
content=TEXT)
if not os.path.exists(indexpath):
os.mkdir(indexpath)
if not exists_in(indexpath):
ix = create_in(indexpath, schema)
else:
ix = open_dir(indexpath)
# LOGGING #
publisher.info("ZMQ Indexer is Running")
while True:
2014-08-14 17:55:18 +02:00
try:
message = p.get_from_set()
2014-08-14 17:55:18 +02:00
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script Indexer is idling 1s")
time.sleep(1)
continue
2014-08-14 17:55:18 +02:00
docpath = message.split(" ", -1)[-1]
paste = PST.get_p_content()
print "Indexing :", docpath
if indexertype == "whoosh":
indexwriter = ix.writer()
2014-08-14 17:55:18 +02:00
indexwriter.update_document(
title=unicode(docpath, errors='ignore'),
path=unicode(docpath, errors='ignore'),
content=unicode(paste, errors='ignore'))
indexwriter.commit()
except IOError:
2014-08-14 17:55:18 +02:00
print "CRC Checksum Failed on :", PST.p_path
2014-08-19 19:07:07 +02:00
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))