From 44addf1afef798bfd4d6ce351029f9fa68cb2db5 Mon Sep 17 00:00:00 2001 From: Starow Date: Fri, 8 Aug 2014 16:48:02 +0200 Subject: [PATCH] Redis cache added fix #5 The paste will be add in Redis during 5min and also saved on disk. Now if a module want to get the paste for further processing, it will first try to get it in the cache instead of getting it directly on the disk and wasting I/O. --- bin/packages/Paste.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index eb953e78..1d59f917 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly: """ -import os, magic, gzip, langid, pprint, redis, operator, string, re, json +import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser from Date import Date from Hash import Hash @@ -30,6 +30,10 @@ from lib_refine import * clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty)) """It filters out non-printable characters from the string it receives.""" +configfile = './config.cfg' +cfg = ConfigParser.ConfigParser() +cfg.read(configfile) + class Paste(object): """ This class representing a Paste as an object. @@ -72,6 +76,11 @@ class Paste(object): self.p_source = var[-5] + self.cache = redis.StrictRedis( + host = cfg.get("Redis_Queues", "host"), + port = cfg.getint("Redis_Queues", "port"), + db = cfg.getint("Redis_Queues", "db")) + def get_p_content(self): """ @@ -82,8 +91,15 @@ class Paste(object): PST.get_p_content() """ - with gzip.open(self.p_path, 'rb') as F: - return F.read() + r_serv = self.cache + + if r_serv.exist(self.p_path): + paste = r_serv.get(self.p_path) + else: + with gzip.open(self.p_path, 'rb') as F: + paste = r_serv.getset(self.p_path, F.read()) + r_serv.expire(self.p_path, 300) + return paste def get_lines_info(self): """