Merge pull request #3479 from FloatingGhost/feature-send-logs-to-elasticsearch

Feature: send logs to elasticsearch

INSTALL/INSTALL.elasticsearch-logging.txt

@@ -0,0 +1,67 @@
+# It's possible to send all logs from MISP to an elasticsearch
+# endpoint
+
+# First, we'll need an ES PHP library
+
+# Replace according to your requirements
+export MISP_DIR=/var/www/MISP
+cd $MISP_DIR/app
+sudo -u www-data php composer.phar require elasticsearch/elasticsearch
+
+# Ok now we need to configure where we log to
+# 
+# In Administration -> Server Settings & Maintenance -> Plugin Settings
+# Under the elasticsearch tab, enable elasticsearch logging, and input
+# your connection string
+# Note that explicitly specifying the port may be needed, e.g. for AWS instances
+# running on 443.
+# Also input a log index - all logs will be thrown at this index.
+
+# Now give ES a template to work from
+cat << EOF > misp_es_template.json 
+{
+  "template": "misp_logging",
+  "mappings": {
+    "log": {
+      "_source": {
+        "enabled": true
+      },
+      "properties": {
+        "Log.email": {
+          "type": "keyword"
+        },
+        "Log.title": {
+          "type": "text"
+        },
+        "Log.ip": {
+          "type": "ip"
+        },
+        "Log.created": {
+          "format": "YYYY-MM-dd HH:mm:ss",
+          "type": "date"
+        },
+        "Log.description": {
+          "type": "text"
+        },
+        "Log.org": {
+          "type": "text"
+        },
+        "Log.action": {
+          "type": "text"
+        },
+        "Log.model": {
+          "type": "text"
+        },
+        "Log.change": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+EOF
+
+# And put it to ES
+curl -XPUT https://my_es/_template/misp_logging --data-binary @misp_es_template.json
+
+# Now MISP will start sending logs to ES! Hooray!

app/Lib/Tools/ElasticSearchClient.php

@@ -0,0 +1,45 @@
+<?php
+
+use Elasticsearch\ClientBuilder;
+
+class ElasticSearchClient {
+    private $__settings = false;
+    private $__client = false;
+
+    private function __getSetSettings() {
+        $settings = array(
+                'enabled' => false,
+                'connection_string' => 'http://localhost',
+        );
+
+        foreach ($settings as $key => $setting) {
+            $temp = Configure::read('Plugin.ElasticSearch_' . $key);
+            if ($temp) $settings[$key] = $temp;
+        }
+        return $settings;
+    }
+
+    public function initTool() {
+        $settings = $this->__getSetSettings();
+        $hosts = explode(",", $settings["connection_string"]);
+        $client = ClientBuilder::create()
+                    ->setHosts($hosts)
+                    ->build();  
+        $this->__client = $client;
+        $this->__settings = $settings;
+        return $client;
+    }
+
+    public function pushDocument($index, $document_type, $document) {
+        // Format timestamp
+        $time = strftime("%Y-%m-%d %H:%M:%S", strtotime($document["Log"]["created"]));
+        $document["Log"]["created"] = $time;
+        $params = array(
+            'index' => $index,
+            'type' => $document_type,
+            'body' => $document
+        );
+
+        $this->__client->index($params);
+    }
+}

app/Model/AppModel.php

@@ -36,6 +36,8 @@ class AppModel extends Model {
 
 	private $__profiler = array();
 
+	public $elasticSearchClient = false;
+
 	public function __construct($id = false, $table = null, $ds = null) {
 		parent::__construct($id, $table, $ds);
 
@@ -1320,6 +1322,20 @@ class AppModel extends Model {
 		return true;
 	}
 
+	public function getElasticSearchTool() {
+		if (!$this->elasticSearchClient) {
+			$this->loadElasticSearchTool();
+		}
+		return $this->elasticSearchClient;
+	}
+
+	public function loadElasticSearchTool() {
+		App::uses('ElasticSearchClient', 'Tools');
+		$client = new ElasticSearchClient();
+		$client->initTool();
+		$this->elasticSearchClient = $client;
+	}
+
 	public function checkVersionRequirements($versionString, $minVersion) {
 		$version = explode('.', $versionString);
 		$minVersion = explode('.', $minVersion);

app/Model/Log.php

@@ -239,6 +239,14 @@ class Log extends AppModel {
 			$pubSubTool = $this->getPubSubTool();
 			$pubSubTool->publish($data, 'audit', 'log');
 		}
+
+		if (Configure::read('Plugin.ElasticSearch_logging_enable')) {
+			// send off our logs to distributed /dev/null
+			$logIndex = Configure::read("Plugin.ElasticSearch_log_index");
+			$elasticSearchClient = $this->getElasticSearchTool();
+			$elasticSearchClient->pushDocument($logIndex, "log", $data);
+        	}
+
 		if (Configure::read('Security.syslog')) {
 			// write to syslogd as well
 			$syslog = new SysLog();

app/Model/Server.php

@@ -1364,6 +1364,30 @@ class Server extends AppModel {
 						'test' => 'testBool',
 						'type' => 'boolean'
 					),
+                    'ElasticSearch_logging_enable' => array (
+                        'level' => 2,
+                        'description' => 'Enabled logging to an ElasticSearch instance',
+                        'value' => false,
+                        'errorMessage' => '',
+                        'test' => 'testBool',
+                        'type' => 'boolean'
+                    ),
+                    'ElasticSearch_connection_string' => array(
+                        'level' => 2,
+                        'description' => 'The URL(s) at which to access ElasticSearch - comma seperate if you want to have more than one.',
+                        'value' => '',
+                        'errorMessage' => '',
+                        'test' => 'testForEmpty',
+                        'type' => 'string'
+                    ),
+                    'ElasticSearch_log_index' => array(
+                        'level' => 2,
+                        'description' => 'The index in which to place logs',
+                        'value' => '',
+                        'errorMessage' => '',
+                        'test' => 'testForEmpty',
+                        'type' => 'string'
+                    ),
 					'Sightings_policy' => array(
 						'level' => 1,
 						'description' => 'This setting defines who will have access to seeing the reported sightings. The default setting is the event owner alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and anyone that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).',

app/composer.json

@@ -5,6 +5,8 @@
         "kamisama/cake-resque": "@stable",
         "pear/crypt_gpg": "@stable",
         "pear/net_geoip": "@dev"
+    },
+    "suggest": {
+        "elasticsearch/elasticsearch": "For logging to elasticsearch"
     }
 }
-