MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'kafka' into 2.4

pull/4346/head
iglocska 2019-03-19 17:23:05 +01:00
parent e8263467df 49be0f8b7a
commit 7141f70b20
32 changed files with 695 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
INSTALL/MYSQL.sql
View File

@ -710,6 +710,7 @@ CREATE TABLE IF NOT EXISTS `roles` (
`max_execution_time` VARCHAR(255) COLLATE utf8_bin DEFAULT "",
`restricted_to_site_admin` tinyint(1) NOT NULL DEFAULT 0,
`perm_publish_zmq` tinyint(1) NOT NULL DEFAULT 0,
`perm_publish_kafka` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
@ -1261,23 +1262,23 @@ INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`,
-- 7. Read Only - read
--
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (1, 'admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (1, 'admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-- --------------------------------------------------------

14
INSTALL/POSTGRESQL-data-initial.sql
View File

@ -309,13 +309,13 @@ COPY public.regexp (id, regexp, replacement, type) FROM stdin;
-- Data for Name: roles; Type: TABLE DATA; Schema: public; Owner: -
--
COPY public.roles (id, name, created, modified, perm_add, perm_modify, perm_modify_org, perm_publish, perm_delegate, perm_sync, perm_admin, perm_audit, perm_full, perm_auth, perm_site_admin, perm_regexp_access, perm_tagger, perm_template, perm_sharing_group, perm_tag_editor, perm_sighting, perm_object_template, default_role, memory_limit, max_execution_time, restricted_to_site_admin, perm_publish_zmq) FROM stdin;
1 admin 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t t t t t t t t t t t t t t f f t
2 Org Admin 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t f t t f t f f t t t t t f f f t
3 User 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t f f f f f f t f f f f f f t f t f f
4 Publisher 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t f f f f t f f f f f f t f f f t
5 Sync user 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t t f f f t f f f f t f t f f f t
6 Read Only 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 f f f f f f f f f t f f f f f f f f f f f
COPY public.roles (id, name, created, modified, perm_add, perm_modify, perm_modify_org, perm_publish, perm_delegate, perm_sync, perm_admin, perm_audit, perm_full, perm_auth, perm_site_admin, perm_regexp_access, perm_tagger, perm_template, perm_sharing_group, perm_tag_editor, perm_sighting, perm_object_template, default_role, memory_limit, max_execution_time, restricted_to_site_admin, perm_publish_zmq, perm_publish_kafka) FROM stdin;
1 admin 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t t t t t t t t t t t t t t f f t t
2 Org Admin 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t f t t f t f f t t t t t f f f t t
3 User 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t f f f f f f t f f f f f f t f t f f f
4 Publisher 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t f f f f t f f f f f f t f f f t t
5 Sync user 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 t t t t t t f f f t f f f f t f t f f f t t
6 Read Only 2018-11-27 06:22:00+00 2018-11-27 06:22:00+00 f f f f f f f f f t f f f f f f f f f f f f
\.

3
INSTALL/POSTGRESQL-structure.sql
View File

@ -1134,7 +1134,8 @@ CREATE TABLE public.roles (
memory_limit character varying(255) DEFAULT ''::character varying,
max_execution_time character varying(255) DEFAULT ''::character varying,
restricted_to_site_admin boolean DEFAULT false NOT NULL,
perm_publish_zmq boolean DEFAULT false NOT NULL
perm_publish_zmq boolean DEFAULT false NOT NULL,
perm_publish_kafka boolean DEFAULT false NOT NULL
);

5
INSTALL/old/INSTALL.debian7.txt
View File

@ -237,3 +237,8 @@ Optional features
-------------------
# MISP has a new pub/sub feature, using ZeroMQ. To enable it, simply run the following command
pip install pyzmq
# MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done

5
INSTALL/old/INSTALL.ubuntu1404.txt
View File

@ -253,3 +253,8 @@ Optional features
pip install pyzmq
# ZeroMQ depends on the Python client for Redis
pip install redis
# MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done

5
INSTALL/old/INSTALL.ubuntu1604.with.webmin.txt
View File

@ -270,6 +270,11 @@ sudo pip install pyzmq
# ZeroMQ depends on the Python client for Redis
sudo pip install redis
# MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done
# For the experimental ssdeep correlations, run the following installation:
# installing ssdeep
wget http://downloads.sourceforge.net/project/ssdeep/ssdeep-2.13/ssdeep-2.13.tar.gz

1
app/Controller/AppController.php
View File

@ -442,6 +442,7 @@ class AppController extends Controller
$this->set('isAclSharingGroup', $role['perm_sharing_group']);
$this->set('isAclSighting', isset($role['perm_sighting']) ? $role['perm_sighting'] : false);
$this->set('isAclZmq', isset($role['perm_publish_zmq']) ? $role['perm_publish_zmq'] : false);
$this->set('isAclKafka', isset($role['perm_publish_kafka']) ? $role['perm_publish_kafka'] : false);
$this->userRole = $role;
} else {
$this->set('me', false);

1
app/Controller/Component/ACLComponent.php
View File

@ -130,6 +130,7 @@ class ACLComponent extends Component
'proposalEventIndex' => array('*'),
'publish' => array('perm_publish'),
'pushEventToZMQ' => array('perm_publish_zmq'),
'pushEventToKafka' => array('perm_publish_kafka'),
'pushProposals' => array('perm_sync'),
'queryEnrichment' => array('perm_add'),
'removePivot' => array('*'),

65
app/Controller/EventsController.php
View File

@ -2364,6 +2364,18 @@ class EventsController extends AppController
$result = $this->Event->save($event, array('fieldList' => $fieldList));
if ($result) {
$message = __('Event unpublished.');
$kafkaTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic');
if (Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaTopic)) {
$kafkaPubTool = $this->Event->getKafkaPubTool();
$params = array('eventid' => $id);
if (Configure::read('Plugin.Kafka_include_attachments')) {
$params['includeAttachments'] = 1;
}
$pubEvent = $this->Event->fetchEvent($this->Auth->user(), $params);
if (!empty($pubEvent)) {
$kafkaPubTool->publishJson($kafkaTopic, $pubEvent[0], 'unpublish');
}
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('events', 'unpublish', $id, false, $message);
} else {
@ -5429,6 +5441,59 @@ class EventsController extends AppController
}
}
public function pushEventToKafka($id)
{
if ($this->request->is('Post')) {
$message = 'Kafka event publishing not enabled.';
if (Configure::read('Plugin.Kafka_enable')) {
$kafkaEventTopic = Configure::read('Plugin.Kafka_event_notifications_topic');
$event = $this->Event->quickFetchEvent(array('eventid' => $id));
if (Configure::read('Plugin.Kafka_event_notifications_enable') && !empty($kafkaEventTopic)) {
$kafkaPubTool = $this->Event->getKafkaPubTool();
if (!empty($event)) {
$kafkaPubTool->publishJson($kafkaEventTopic, $event, 'manual_publish');
$success = 1;
$message = 'Event published to Kafka';
} else {
$success = 0;
$message = 'Invalid event.';
}
}
$kafkaPubTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic');
if (!empty($event['Event']['published']) && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaPubTopic)) {
$kafkaPubTool = $this->Event->getKafkaPubTool();
$params = array('eventid' => $id);
if (Configure::read('Plugin.Kafka_include_attachments')) {
$params['includeAttachments'] = 1;
}
$event = $this->Event->fetchEvent($this->Auth->user(), $params);
if (!empty($event)) {
$kafkaPubTool->publishJson($kafkaPubTopic, $event[0], 'manual_publish');
if (!isset($success)) {
$success = 1;
$message = 'Event published to Kafka';
}
} else {
$success = 0;
$message = 'Invalid event.';
}
}
}
} else {
$message = 'This functionality is only available via POST requests';
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Events', 'pushEventToKafka', $id, $this->response->type(), $message);
} else {
if (!empty($success)) {
$this->Flash->success($message);
} else {
$this->Flash->error($message);
}
$this->redirect($this->referer());
}
}
public function getEventInfoById($id)
{
if (empty($id)) {

2
app/Controller/ShadowAttributesController.php
View File

@ -68,6 +68,7 @@ class ShadowAttributesController extends AppController
if (empty($shadow)) {
return array('false' => true, 'errors' => 'Proposal not found or you are not authorised to accept it.');
}
$this->ShadowAttribute->publishKafkaNotification('shadow_attribute', $shadow, 'accept');
$shadow = $shadow['ShadowAttribute'];
if ($this->ShadowAttribute->typeIsAttachment($shadow['type'])) {
$encodedFile = $this->ShadowAttribute->base64EncodeAttachment($shadow);
@ -229,6 +230,7 @@ class ShadowAttributesController extends AppController
if (empty($sa)) {
return false;
}
$this->ShadowAttribute->publishKafkaNotification('shadow_attribute', $sa, 'discard');
$eventId = $sa['ShadowAttribute']['event_id'];
$this->loadModel('Event');
$this->Event->Behaviors->detach('SysLogLogable.SysLogLogable');

55
app/Lib/Tools/KafkaPubTool.php Normal file
View File

@ -0,0 +1,55 @@
<?php
class KafkaPubTool
{
private $rdkafka = false;
private function __error($msg)
{
error_log($msg, 3, APP . 'tmp' . DS . 'logs' . DS . 'kafka.error.log');
}
public function initTool($brokers, $config)
{
if (!$this->rdkafka) {
try {
$rdConf = new RdKafka\Conf();
foreach ($config as $key => $val) {
if (!empty($val)) {
$rdConf->set($key, $val);
}
}
$rdConf->setErrorCb(function ($kafka, $err, $reason) {
$this->__error(sprintf("%s (reason: %s)\n", rd_kafka_err2str($err), $reason));
});
$rdkafka = new RdKafka\Producer($rdConf);
if ($rdkafka->addBrokers($brokers) == 0) {
$this->__error("Could not add any Kafka brokers");
}
$this->rdkafka = $rdkafka;
} catch (Exception $e) {
$this->__error('Exception: ' . $e->getMessage() . "\n");
}
}
}
public function publishJson($topicName, $data, $action = false)
{
try {
if (!empty($action)) {
$data['action'] = $action;
}
$body = json_encode($data);
if (!$body) {
$this->__error("Error encoding to JSON: ". $data);
}
if (!empty($this->rdkafka)) {
$topic = $this->rdkafka->newTopic($topicName);
$topic->produce(RD_KAFKA_PARTITION_UA, 0, $body);
$this->rdkafka->poll(0);
}
} catch (Exception $e) {
$this->__error('Exception: ' . $e->getMessage() . "\n");
}
}
}

38
app/Model/AppModel.php
View File

@ -30,6 +30,8 @@ class AppModel extends Model
public $loadedPubSubTool = false;
public $loadedKafkaPubTool = false;
public $start = 0;
public $inserted_ids = array();
@ -72,7 +74,8 @@ class AppModel extends Model
7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false,
13 => false, 14 => false, 15 => false, 18 => false, 19 => false, 20 => false,
21 => false, 22 => false, 23 => false, 24 => false, 25 => false, 26 => false,
27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false
27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false,
33 => false
);
public function afterSave($created, $options = array())
@ -1121,6 +1124,9 @@ class AppModel extends Model
case 32:
$sqlArray[] = "ALTER TABLE `taxonomies` ADD `required` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 33:
$sqlArray[] = "ALTER TABLE `roles` ADD `perm_publish_kafka` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -1531,6 +1537,36 @@ class AppModel extends Model
return $redis;
}
public function getKafkaPubTool()
{
if (!$this->loadedKafkaPubTool) {
$this->loadKafkaPubTool();
}
return $this->loadedKafkaPubTool;
}
public function loadKafkaPubTool()
{
App::uses('KafkaPubTool', 'Tools');
$kafkaPubTool = new KafkaPubTool();
$rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config');
$kafkaConf = array();
if (!empty($rdkafkaIni)) {
$kafkaConf = parse_ini_file($rdkafkaIni);
}
$brokers = Configure::read('Plugin.Kafka_brokers');
$kafkaPubTool->initTool($brokers, $kafkaConf);
$this->loadedKafkaPubTool = $kafkaPubTool;
return true;
}
public function publishKafkaNotification($topicName, $data, $action = false) {
$kafkaTopic = Configure::read('Plugin.Kafka_' . $topicName . '_notifications_topic');
if (Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_' . $topicName . '_notifications_enable') && !empty($kafkaTopic)) {
$this->getKafkaPubTool()->publishJson($kafkaTopic, $data, $action);
}
}
public function getPubSubTool()
{
if (!$this->loadedPubSubTool) {

28
app/Model/Attribute.php
View File

@ -645,8 +645,10 @@ class Attribute extends AppModel
if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type']) && !empty($this->data['Attribute']['data'])) {
$result = $result && $this->saveBase64EncodedAttachment($this->data['Attribute']); // TODO : is this correct?
}
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_attribute_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_attribute_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$attribute = $this->fetchAttribute($this->id);
if (!empty($attribute)) {
$user = array(
@ -663,10 +665,21 @@ class Attribute extends AppModel
if (!empty($this->data['Attribute']['deleted'])) {
$action = 'soft-delete';
}
if (Configure::read('Plugin.ZeroMQ_include_attachments') && $this->typeIsAttachment($attribute['Attribute']['type'])) {
$attribute['Attribute']['data'] = $this->base64EncodeAttachment($attribute['Attribute']);
if ($pubToZmq) {
if (Configure::read('Plugin.ZeroMQ_include_attachments') && $this->typeIsAttachment($attribute['Attribute']['type'])) {
$attribute['Attribute']['data'] = $this->base64EncodeAttachment($attribute['Attribute']);
}
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($attribute, $action);
unset($attribute['Attribute']['data']);
}
if ($pubToKafka) {
if (Configure::read('Plugin.Kafka_include_attachments') && $this->typeIsAttachment($attribute['Attribute']['type'])) {
$attribute['Attribute']['data'] = $this->base64EncodeAttachment($attribute['Attribute']);
}
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $attribute, $action);
}
$pubSubTool->attribute_save($attribute, $action);
}
}
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
@ -712,6 +725,11 @@ class Attribute extends AppModel
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($this->data, 'delete');
}
$kafkaTopic = Configure::read('Plugin.Kafka_attribute_notifications_topic');
if (Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_attribute_notifications_enable') && !empty($kafkaTopic)) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $this->data, 'delete');
}
}
}

30
app/Model/AttributeTag.php
View File

@ -30,8 +30,10 @@ class AttributeTag extends AppModel
public function afterSave($created, $options = array())
{
parent::afterSave($created, $options);
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$tag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('AttributeTag.id' => $this->id),
@ -40,15 +42,24 @@ class AttributeTag extends AppModel
$tag['Tag']['attribute_id'] = $tag['AttributeTag']['attribute_id'];
$tag['Tag']['event_id'] = $tag['AttributeTag']['event_id'];
$tag = array('Tag' => $tag['Tag']);
$pubSubTool->tag_save($tag, 'attached to attribute');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->tag_save($tag, 'attached to attribute');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $tag, 'attached to attribute');
}
}
}
public function beforeDelete($cascade = true)
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable')) {
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
if (!empty($this->id)) {
$pubSubTool = $this->getPubSubTool();
$tag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('AttributeTag.id' => $this->id),
@ -57,7 +68,14 @@ class AttributeTag extends AppModel
$tag['Tag']['attribute_id'] = $tag['AttributeTag']['attribute_id'];
$tag['Tag']['event_id'] = $tag['AttributeTag']['event_id'];
$tag = array('Tag' => $tag['Tag']);
$pubSubTool->tag_save($tag, 'detached from attribute');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->tag_save($tag, 'detached from attribute');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $tag, 'detached from attribute');
}
}
}
}

59
app/Model/Event.php
View File

@ -358,6 +358,29 @@ class Event extends AppModel
$pubSubTool = $this->getPubSubTool();
$pubSubTool->event_save(array('Event' => $this->data['Event']), 'delete');
}
if (Configure::read('Plugin.Kafka_enable')) {
$kafkaEventTopic = Configure::read('Plugin.Kafka_event_notifications_topic');
if(Configure::read('Plugin.Kafka_event_notifications_enable') && !empty($kafkaEventTopic)) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaEventTopic, array('Event' => $this->data['Event']), 'delete');
}
$kafkaPubTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic');
if (!empty($this->data['Event']['published']) && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaPubTopic)) {
$hostOrg = $this->Org->find('first', array('conditions' => array('name' => Configure::read('MISP.org')), 'fields' => array('id')));
if (!empty($hostOrg)) {
$user = array('org_id' => $hostOrg['Org']['id'], 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']);
$params = array('eventid' => $this->data['Event']['id']);
if (Configure::read('Plugin.Kafka_include_attachments')) {
$params['includeAttachments'] = 1;
}
$fullEvent = $this->fetchEvent($user, $params);
if (!empty($fullEvent)) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaPubTopic, $fullEvent[0], 'delete');
}
}
}
}
}
}
@ -483,6 +506,9 @@ class Event extends AppModel
$pubSubTool->event_save($event, $created ? 'add' : 'edit');
}
}
if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_kafka'])) {
$this->publishKafkaNotification('event', $this->quickFetchEvent($this->data['Event']['id']), $created ? 'add' : 'edit');
}
}
public function buildEventConditions($user)
@ -3874,20 +3900,37 @@ class Event extends AppModel
$event['Event']['published'] = 1;
$event['Event']['publish_timestamp'] = time();
$event['Event']['skip_zmq'] = 1;
$event['Event']['skip_kafka'] = 1;
$this->save($event, array('fieldList' => $fieldList));
}
if (Configure::read('Plugin.ZeroMQ_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$hostOrg = $this->Org->find('first', array('conditions' => array('name' => Configure::read('MISP.org')), 'fields' => array('id')));
if (!empty($hostOrg)) {
$user = array('org_id' => $hostOrg['Org']['id'], 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']);
$params = array('eventid' => $id);
if (Configure::read('Plugin.ZeroMQ_include_attachments')) {
$params['includeAttachments'] = 1;
if ($pubToZmq) {
$params = array('eventid' => $id);
if (Configure::read('Plugin.ZeroMQ_include_attachments')) {
$params['includeAttachments'] = 1;
}
$fullEvent = $this->fetchEvent($user, $params);
if (!empty($fullEvent)) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->publishEvent($fullEvent[0], 'publish');
}
}
$fullEvent = $this->fetchEvent($user, $params);
if (!empty($fullEvent)) {
$pubSubTool->publishEvent($fullEvent[0], 'publish');
if ($pubToKafka) {
$params = array('eventid' => $id);
if (Configure::read('Plugin.Kafka_include_attachments')) {
$params['includeAttachments'] = 1;
}
$fullEvent = $this->fetchEvent($user, $params);
if (!empty($fullEvent)) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $fullEvent[0], 'publish');
}
}
}
}

30
app/Model/EventTag.php
View File

@ -26,8 +26,10 @@ class EventTag extends AppModel
public function afterSave($created, $options = array())
{
parent::afterSave($created, $options);
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$tag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('EventTag.id' => $this->id),
@ -35,15 +37,24 @@ class EventTag extends AppModel
));
$tag['Tag']['event_id'] = $tag['EventTag']['event_id'];
$tag = array('Tag' => $tag['Tag']);
$pubSubTool->tag_save($tag, 'attached to event');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->tag_save($tag, 'attached to event');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $tag, 'attached to event');
}
}
}
public function beforeDelete($cascade = true)
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable')) {
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
if (!empty($this->id)) {
$pubSubTool = $this->getPubSubTool();
$tag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('EventTag.id' => $this->id),
@ -51,7 +62,14 @@ class EventTag extends AppModel
));
$tag['Tag']['event_id'] = $tag['EventTag']['event_id'];
$tag = array('Tag' => $tag['Tag']);
$pubSubTool->tag_save($tag, 'detached from event');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->tag_save($tag, 'detached from event');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $tag, 'detached from event');
}
}
}
}

2
app/Model/Log.php
View File

@ -261,6 +261,8 @@ class Log extends AppModel
$pubSubTool->publish($data, 'audit', 'log');
}
$this->publishKafkaNotification('audit', $data, 'log');
if (Configure::read('Plugin.ElasticSearch_logging_enable')) {
// send off our logs to distributed /dev/null
$logIndex = Configure::read("Plugin.ElasticSearch_log_index");

48
app/Model/MispObject.php
View File

@ -92,19 +92,31 @@ class MispObject extends AppModel
public function afterSave($created, $options = array())
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable')) {
if (empty($this->data['Object']['skip_zmq'])) {
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') &&
Configure::read('Plugin.ZeroMQ_object_notifications_enable') &&
empty($this->data['Object']['skip_zmq']);
$kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') &&
Configure::read('Plugin.Kafka_object_notifications_enable') &&
!empty($kafkaTopic) &&
empty($this->data['Object']['skip_kafka']);
if ($pubToZmq || $pubToKafka) {
$object = $this->find('first', array(
'conditions' => array('Object.id' => $this->id),
'recursive' => -1
));
$action = $created ? 'add' : 'edit';
if (!empty($this->data['Object']['deleted'])) {
$action = 'soft-delete';
}
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$object = $this->find('first', array(
'conditions' => array('Object.id' => $this->id),
'recursive' => -1
));
$action = $created ? 'add' : 'edit';
if (!empty($this->data['Object']['deleted'])) {
$action = 'soft-delete';
}
$pubSubTool->object_save($object, $action);
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $object, $action);
}
}
return true;
}
@ -112,13 +124,22 @@ class MispObject extends AppModel
public function beforeDelete($cascade = true)
{
if (!empty($this->data['Object']['id'])) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $this->data['Object']['id'])
));
$pubSubTool->object_save($object, 'delete');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->object_save($object, 'delete');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $object, 'delete');
}
}
}
}
@ -686,6 +707,7 @@ class MispObject extends AppModel
));
$object['Object']['timestamp'] = $date->getTimestamp();
$object['Object']['skip_zmq'] = 1;
$object['Object']['skip_kafka'] = 1;
$result = $this->save($object);
return $result;
}

15
app/Model/ObjectReference.php
View File

@ -55,8 +55,10 @@ class ObjectReference extends AppModel
public function afterSave($created, $options = array())
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_reference_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_reference_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_object_reference_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_reference_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$object_reference = $this->find('first', array(
'conditions' => array('ObjectReference.id' => $this->id),
'recursive' => -1
@ -65,7 +67,14 @@ class ObjectReference extends AppModel
if (!empty($this->data['ObjectReference']['deleted'])) {
$action = 'soft-delete';
}
$pubSubTool->object_reference_save($object_reference, $action);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->object_reference_save($object_reference, $action);
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $object_reference, $action);
}
}
return true;
}

2
app/Model/Organisation.php
View File

@ -125,6 +125,8 @@ class Organisation extends AppModel
$pubSubTool = $this->getPubSubTool();
$pubSubTool->modified($this->data, 'organisation');
}
$action = $created ? 'add' : 'edit';
$this->publishKafkaNotification('organisation', $this->data, $action);
return true;
}

6
app/Model/Role.php
View File

@ -140,6 +140,12 @@ class Role extends AppModel
'text' => 'ZMQ publisher',
'readonlyenabled' => false,
'title' => 'Allow users to publish data to the ZMQ pubsub channel via the publish event to ZMQ button.'
),
'perm_publish_kafka' => array(
'id' => 'RolePermPublishKafka',
'text' => 'Kafka publisher',
'readonlyenabled' => false,
'title' => 'Allow users to publish data to Kafka via the publish event to Kafka button.'
)
);

208
app/Model/Server.php
View File

@ -1288,6 +1288,214 @@ class Server extends AppModel
'test' => 'testForEmpty',
'type' => 'string',
),
'Kafka_enable' => array(
'level' => 2,
'description' => __('Enables or disables the Kafka pub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
),
'Kafka_brokers' => array(
'level' => 2,
'description' => __('A comma separated list of Kafka bootstrap brokers'),
'value' => 'kafka:9092',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
),
'Kafka_rdkafka_config' => array(
'level' => 2,
'description' => __('A path to an ini file with configuration options to be passed to rdkafka. Section headers in the ini file will be ignored.'),
'value' => '/etc/rdkafka.ini',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
),
'Kafka_include_attachments' => array(
'level' => 2,
'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_event_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_event_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing event creations/edits/deletions.'),
'value' => 'misp_event',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_event_publish_notifications_enable' => array(
'level' => 2,
'description' => __('If enabled it will publish to Kafka the event at the time that the event gets published in MISP. Event actions (creation or edit) will not be published to Kafka.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_event_publish_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing event information on publish.'),
'value' => 'misp_event_publish',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_object_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_object_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing object creations/edits/deletions.'),
'value' => 'misp_object',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_object_reference_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of any object reference creations/deletions.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_object_reference_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing object reference creations/deletions.'),
'value' => 'misp_object_reference',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_attribute_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_attribute_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing attribute creations/edits/soft deletions.'),
'value' => 'misp_attribute',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_shadow_attribute_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of any proposal creations/edits/deletions.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_shadow_attribute_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing proposal creations/edits/deletions.'),
'value' => 'misp_shadow_attribute',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_tag_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_tag_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'),
'value' => 'misp_tag',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_sighting_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of new sightings.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_sighting_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing sightings.'),
'value' => 'misp_sighting',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_user_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of new/modified users.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_user_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing new/modified users.'),
'value' => 'misp_user',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_organisation_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of new/modified organisations.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_organisation_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing new/modified organisations.'),
'value' => 'misp_organisation',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Kafka_audit_notifications_enable' => array(
'level' => 2,
'description' => __('Enables or disables the publishing of log entries. Keep in mind, this can get pretty verbose depending on your logging settings.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'Kafka_audit_notifications_topic' => array(
'level' => 2,
'description' => __('Topic for publishing log entries.'),
'value' => 'misp_audit',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'ZeroMQ_enable' => array(
'level' => 2,
'description' => __('Enables or disables the pub/sub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'),

5
app/Model/ShadowAttribute.php
View File

@ -277,6 +277,10 @@ class ShadowAttribute extends AppModel
} else {
$this->__afterSaveCorrelation($this->data['ShadowAttribute']);
}
if (empty($this->data['ShadowAttribute']['deleted'])) {
$action = $created ? 'add' : 'edit';
$this->publishKafkaNotification('shadow_attribute', $this->data, $action);
}
return $result;
}
@ -555,6 +559,7 @@ class ShadowAttribute extends AppModel
}
$fieldList = array('proposal_email_lock', 'id', 'info');
$event['Event']['skip_zmq'] = 1;
$event['Event']['skip_kafka'] = 1;
$this->Event->save($event, array('fieldList' => $fieldList));
}

30
app/Model/Sighting.php
View File

@ -60,8 +60,10 @@ class Sighting extends AppModel
public function afterSave($created, $options = array())
{
parent::afterSave($created, $options = array());
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_sighting_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_sighting_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$user = array(
'org_id' => -1,
'Role' => array(
@ -69,7 +71,14 @@ class Sighting extends AppModel
)
);
$sighting = $this->getSighting($this->id, $user);
$pubSubTool->sighting_save($sighting, 'add');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($sighting, 'add');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $sighting, 'add');
}
}
return true;
}
@ -77,8 +86,10 @@ class Sighting extends AppModel
public function beforeDelete($cascade = true)
{
parent::beforeDelete();
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_sighting_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_sighting_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$user = array(
'org_id' => -1,
'Role' => array(
@ -86,7 +97,14 @@ class Sighting extends AppModel
)
);
$sighting = $this->getSighting($this->id, $user);
$pubSubTool->sighting_save($sighting, 'delete');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($sighting, 'delete');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $sighting, 'delete');
}
}
}

30
app/Model/Tag.php
View File

@ -90,27 +90,45 @@ class Tag extends AppModel
public function afterSave($created, $options = array())
{
parent::afterSave($created, $options);
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$tag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Tag.id' => $this->id)
));
$action = $created ? 'add' : 'edit';
$pubSubTool->tag_save($tag, $action);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->tag_save($tag, $action);
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $tag, $action);
}
}
}
public function beforeDelete($cascade = true)
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable')) {
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_tag_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_tag_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_tag_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
if (!empty($this->id)) {
$pubSubTool = $this->getPubSubTool();
$tag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Tag.id' => $this->id)
));
$pubSubTool->tag_save($tag, 'delete');
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->tag_save($tag, 'delete');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $tag, 'delete');
}
}
}
}

15
app/Model/User.php
View File

@ -271,8 +271,10 @@ class User extends AppModel
public function afterSave($created, $options = array())
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_user_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_user_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_user_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_user_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
if (!empty($this->data)) {
$user = $this->data;
if (!isset($user['User'])) {
@ -298,7 +300,14 @@ class User extends AppModel
unset($user['User']['password']);
unset($user['User']['confirm_password']);
}
$pubSubTool->modified($user, 'user', $action);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->modified($user, 'user', $action);
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $user, $action);
}
}
}
return true;

10
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -181,6 +181,16 @@
'message' => __('Are you sure you wish to republish the current event to the ZMQ channel?')
));
}
if (Configure::read('Plugin.Kafka_enable') &&
Configure::read('Plugin.Kafka_event_notifications_enable') &&
Configure::read('Plugin.Kafka_event_notifications_topic') &&
$isAclKafka) {
echo $this->element('/genericElements/SideMenu/side_menu_post_link', array(
'url' => '/events/pushEventToKafka/' . h($event['Event']['id']),
'text' => __('Publish event to Kafka'),
'message' => __('Are you sure you wish to republish the current event to the Kafka topic?')
));
}
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'contact',
'url' => '/events/contact/' . $event['Event']['id'],

7
docs/INSTALL.ubuntu1804.md
View File

@ -473,6 +473,13 @@ echo "User (misp) DB Password: $DBPASSWORD_MISP"
sudo -H -u www-data ${PATH_TO_MISP}/venv/bin/pip install pyzmq
```
#### MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
```bash
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done
```
{!generic/misp-dashboard-debian.md!}
{!generic/viper-debian.md!}

7
docs/archive/INSTALL.ubuntu1604.md
View File

@ -441,6 +441,13 @@ echo "User (misp) DB Password: $DBPASSWORD_MISP"
sudo -H -u www-data /var/www/MISP/venv/bin/pip install pyzmq
```
#### MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
```bash
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done
```
!!! warning
There is an issue with the apache config of misp-dashboard in Ubuntu 16.04
You need to **remove** the following 3 options from **WSGIDaemonProcess**

7
docs/xINSTALL.debian9.md
View File

@ -490,6 +490,13 @@ In case you are using a virtualenv make sure pyzmq is installed therein.
sudo -u www-data ${PATH_TO_MISP}/venv/bin/pip install pyzmq
```
#### MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
```bash
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done
```
{!generic/misp-dashboard-debian.md!}
{!generic/viper-debian.md!}

7
docs/xINSTALL.debian_testing.md
View File

@ -486,6 +486,13 @@ In case you are using a virtualenv make sure pyzmq is installed therein.
sudo -u www-data ${PATH_TO_MISP}/venv/bin/pip install pyzmq
```
#### MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
```bash
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done
```
{!generic/misp-dashboard-debian.md!}
{!generic/viper-debian.md!}

7
docs/xINSTALL.ubuntu1804.with.webmin.md
View File

@ -444,6 +444,13 @@ sudo pip3 install pyzmq
sudo pip3 install redis
```
#### MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
```bash
apt-get install librdkafka-dev php-dev
pecl install rdkafka
find /etc -name php.ini | while read f; do echo 'extension=rdkafka.so' | tee -a "$f"; done
```
#### Experimental ssdeep correlations
```bash
# installing ssdeep