mirror of https://github.com/CIRCL/AIL-framework
chg: [IPAddress module] migrate IPAddress module
Terrtia
parent
7669c16c74
commit
ed9682798e
|
|
@ -1,90 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The IP Module
|
||||
======================
|
||||
|
||||
This module is consuming the global channel.
|
||||
|
||||
It first performs a regex to find IP addresses and then matches those IPs to
|
||||
some configured ip ranges.
|
||||
|
||||
The list of IP ranges are expected to be in CIDR format (e.g. 192.168.0.0/16)
|
||||
and should be defined in the config.cfg file, under the [IP] section
|
||||
|
||||
"""
|
||||
|
||||
import time
|
||||
import re
|
||||
import sys
|
||||
from pubsublogger import publisher
|
||||
from lib.objects.Items import Item
|
||||
from Helper import Process
|
||||
from ipaddress import IPv4Network, IPv4Address
|
||||
|
||||
# TODO REWRITE ME -> IMPROVE + MIGRATE TO MODULE
|
||||
|
||||
def search_ip(message):
|
||||
item = Item(message)
|
||||
content = item.get_content()
|
||||
# regex to find IPs
|
||||
reg_ip = re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', flags=re.MULTILINE)
|
||||
# list of the regex results in the Item, may be null
|
||||
results = reg_ip.findall(content)
|
||||
matching_ips = []
|
||||
|
||||
for ip in results:
|
||||
ip = '.'.join([str(int(x)) for x in ip.split('.')])
|
||||
address = IPv4Address(ip)
|
||||
for network in ip_networks:
|
||||
if address in network:
|
||||
matching_ips.append(address)
|
||||
|
||||
if len(matching_ips) > 0:
|
||||
print(f'{item.get_id()} contains {len(matching_ips)} IPs')
|
||||
publisher.warning(f'{item.get_id()} contains {item.get_id()} IPs')
|
||||
|
||||
# Tag message with IP
|
||||
msg = f'infoleak:automatic-detection="ip";{item.get_id()}'
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'IP'
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
ip_networks = []
|
||||
networks = p.config.get("IP", "networks")
|
||||
if not networks:
|
||||
print('No IP ranges provided')
|
||||
sys.exit(0)
|
||||
try:
|
||||
for network in networks.split(","):
|
||||
ip_networks.append(IPv4Network(network))
|
||||
print(f'IP Range: {network}')
|
||||
except:
|
||||
print('Please provide a list of valid IP addresses')
|
||||
sys.exit(0)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Run IP module")
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# Do something with the message from the queue
|
||||
search_ip(message)
|
||||
|
|
@ -237,6 +237,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Iban" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Iban.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./IPAddress.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Keys.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Languages" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Languages.py; read x"
|
||||
|
|
@ -301,7 +303,6 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
# screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
# sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The IP Module
|
||||
======================
|
||||
|
||||
This module is consuming the global channel.
|
||||
|
||||
It first performs a regex to find IP addresses and then matches those IPs to
|
||||
some configured ip ranges.
|
||||
|
||||
The list of IP ranges are expected to be in CIDR format (e.g. 192.168.0.0/16)
|
||||
and should be defined in the config.cfg file, under the [IP] section
|
||||
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from ipaddress import IPv4Network, IPv4Address
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import regex_helper
|
||||
|
||||
# TODO REWRITE ME -> PERF + IPV6 + Tracker ?
|
||||
|
||||
class IPAddress(AbstractModule):
|
||||
"""Telegram module for AIL framework"""
|
||||
|
||||
def __init__(self):
|
||||
super(IPAddress, self).__init__()
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
# Config Load ip_networks
|
||||
self.ip_networks = set()
|
||||
networks = config_loader.get_config_str("IP", "networks")
|
||||
if not networks:
|
||||
print('No IP ranges provided')
|
||||
sys.exit(0)
|
||||
try:
|
||||
for network in networks.split(","):
|
||||
self.ip_networks.add(IPv4Network(network))
|
||||
print(f'IP Range To Search: {network}')
|
||||
except:
|
||||
print('Please provide a list of valid IP addresses')
|
||||
sys.exit(0)
|
||||
|
||||
self.re_ipv4 = r'(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
|
||||
re.compile(self.re_ipv4)
|
||||
|
||||
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
|
||||
self.max_execution_time = 60
|
||||
|
||||
# Send module state to logs
|
||||
self.logger.info(f"Module {self.module_name} initialized")
|
||||
|
||||
def compute(self, message, r_result=False):
|
||||
item = Item(message)
|
||||
content = item.get_content()
|
||||
|
||||
# list of the regex results in the Item
|
||||
results = self.regex_findall(self.re_ipv4, item.get_id(), content)
|
||||
results = set(results)
|
||||
matching_ips = []
|
||||
for ip in results:
|
||||
ip = '.'.join([str(int(x)) for x in ip.split('.')])
|
||||
address = IPv4Address(ip)
|
||||
for network in self.ip_networks:
|
||||
if address in network:
|
||||
self.logger.info(address)
|
||||
matching_ips.append(address)
|
||||
|
||||
if len(matching_ips) > 0:
|
||||
self.logger.info(f'{item.get_id()} contains {len(matching_ips)} IPs')
|
||||
self.redis_logger.warning(f'{item.get_id()} contains {item.get_id()} IPs')
|
||||
|
||||
# Tag message with IP
|
||||
msg = f'infoleak:automatic-detection="ip";{item.get_id()}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
module = IPAddress()
|
||||
module.run()
|
||||
# module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz')
|
||||
|
|
@ -156,7 +156,7 @@ publish = Importers
|
|||
[Crawler]
|
||||
publish = Importers,Tags
|
||||
|
||||
[IP]
|
||||
[IPAddress]
|
||||
subscribe = Item
|
||||
publish = Tags
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue