diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 684af83b..c751a439 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -170,6 +170,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' sleep 0.1 + screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Regex.py; read x' + sleep 0.1 screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' sleep 0.1 screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' diff --git a/bin/Regex.py b/bin/Regex.py new file mode 100644 index 00000000..960c33be --- /dev/null +++ b/bin/Regex.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +The Regex Module +================ + +Search for regular expressions stored in a flat file + tag. +The flat file is automatically reloaded when the MTIME changed. + +It uses the file 'packagess/regex.cfg'. Format: +Tag||Regex + +Xavier Mertens + +""" + +import time +import os +import re +from pubsublogger import publisher + +#from bin.packages import Paste +#from bin.Helper import Process + +from packages import Paste +from Helper import Process + +# Change the path to your preferred one +regexConfig = 'packages/regex.cfg' + +regexes = [] + +def load_regex(force = False): + ''' + Load regexes from the config file and validate them + If 'True' passed as argument, force to reload + ''' + + lregexes = regexes + validate_regex = False + + try: + stats = os.stat(regexConfig) + mtime = int(stats.st_mtime) + if mtime > time.time()-60 or force == True: + # Regex config changed, reload the file + print('Loading regular expressions') + with open(regexConfig) as f: + lines = f.readlines() + lines = [x.strip() for x in lines] + validate_regex = True + except: + print('Cannot read {}'.format(regexConfig)) + return [] + + if validate_regex: + # Validate regexes read from the file + line=1 + lregexes = [] + for l in lines: + # Skip comments and empty lines + if len(l) > 0: + if l[0] == '#': + continue + try: + re.compile(l.split('||')[1]) + except: + print('Ignored line {}: Syntax error in "{}"'.format(line, regexConfig)) + continue + line += 1 + lregexes.append(l) + print('DEBUG: regexes:') + print(lregexes) + return lregexes + +def search_regex(paste): + content = paste.get_p_content() + find = False + global regexes + + regexes = load_regex(False) + + for r in regexes: + (tag,pattern) = r.split('||') + + if re.findall(pattern, content, re.MULTILINE|re.IGNORECASE): + publisher.warning('Regex match: {} ({})'.format(pattern, tag)) + # Sanitize tag to make it easy to read + tag = tag.strip().lower().replace(' ','-') + print('regex {} found'.format(tag)) + msg = 'infoleak:automatic-detection="regex-{}";{}'.format(tag, message) + p.populate_set_out(msg, 'Tags') + find = True + + if find: + #Send to duplicate + p.populate_set_out(message, 'Duplicate') + #send to Browse_warning_paste + msg = ('regex;{}'.format(message)) + print(message) + p.populate_set_out( msg, 'alertHandler') + + +if __name__ == '__main__': + global regexes + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'Regex' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("Run Regex module ") + + # Load regular expressions from config file + regexes = load_regex(True) + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + # Do something with the message from the queue + paste = Paste.Paste(message) + search_regex(paste) + + # (Optional) Send that thing to the next queue diff --git a/bin/packages/regex.cfg b/bin/packages/regex.cfg new file mode 100644 index 00000000..dcc939c6 --- /dev/null +++ b/bin/packages/regex.cfg @@ -0,0 +1,17 @@ +# +# Regular expressions to be search in AIL +# +# Format: +# tag||regex +# +# tag: appened to the tag (ex: regex-tag) +# regex: the regular expression +# (Comments & empty lines are ignores) +# + +# Example1 +# Search for my name +personal-mention||xavier\s+mertens + +# Search for cicl.lu hostnames +circl||\w+\.circl\.lu