Merge branch '2.4' of https://github.com/MISP/MISP into rework_modules

2019-05-23 14:45:07 +02:00 · 2019-05-23 14:45:07 +02:00 · 543dc04d7a
parent e974a651b2 f1295d7c16
commit 543dc04d7a
12 changed files with 206 additions and 77 deletions
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@ -1081,7 +1081,7 @@ EOF

  sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE DATABASE ${DBNAME};"
  sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
-  sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT USAGE ON *.* to ${DBNAME}@localhost;"
+  sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT USAGE ON *.* to ${DBUSER_MISP}@localhost;"
  sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
  sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "FLUSH PRIVILEGES;"
  # Import the empty MISP database from MYSQL.sql
@ -2156,7 +2156,7 @@ installMISPonKali () {
      expect eof" | expect -f -

    mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;"
-    mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE ON *.* TO $DBNAME@localhost IDENTIFIED BY '$DBPASSWORD_MISP';"
+    mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE ON *.* TO $DBUSER_MISP@localhost IDENTIFIED BY '$DBPASSWORD_MISP';"
    mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT ALL PRIVILEGES ON $DBNAME.* TO '$DBUSER_MISP'@'localhost';"
    mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "FLUSH PRIVILEGES;"

--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit e8bba395bc67bf56e41ddd022ebae670c5b0d64b
+Subproject commit 583fb6592495ea358aad47a8a1ec92d43c13348a
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@ -468,7 +468,10 @@ class AdminShell extends AppShell
        $logs = $this->Log->find('all', array(
            'conditions' => array(
                'action' => 'update_database',
-                'title LIKE ' => 'Successfuly executed the SQL query for %'
+                'title LIKE ' => array(
+                    'Successfuly executed the SQL query for %',
+                    'Issues executing the SQL query for %'
+                )
            ),
            'order' => 'id DESC'
        ));
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@ -2005,7 +2005,7 @@ class AttributesController extends AppController
            'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
            'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
            'includeProposals', 'returnFormat', 'published', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
-            'includeWarninglistHits', 'attackGalaxy'
+            'includeWarninglistHits', 'attackGalaxy', 'object_relation'
        );
        $filterData = array(
            'request' => $this->request,
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@ -421,10 +421,16 @@ class RestResponseComponent extends Component
        } elseif (strtolower($format) == 'csv') {
            $type = 'csv';
        } else {
-            if (!$raw) {
-                $response = json_encode($response, JSON_PRETTY_PRINT);
+            if (empty($format)) {
+                $type = 'json';
+            } else {
+                $type = $format;
+            }
+            if (!$raw) {
+                if (!empty($response)) {
+                    $response = json_encode($response, JSON_PRETTY_PRINT);
+                }
            }
-            $type = 'json';
        }
        $cakeResponse = new CakeResponse(array('body'=> $response, 'status' => $code, 'type' => $type));

--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@ -1057,7 +1057,7 @@ class EventsController extends AppController
            $conditions['overrideLimit'] = 1;
        }
        if (isset($filters['deleted'])) {
-            $conditions['deleted'] = $filters['deleted'] == 2 ? 0 : 1;
+            $conditions['deleted'] = $filters['deleted'] == 2 ? 0 : [0, 1];
        }
        if (isset($filters['toIDS']) && $filters['toIDS'] != 0) {
            $conditions['to_ids'] = $filters['toIDS'] == 2 ? 0 : 1;
@ -1509,7 +1509,7 @@ class EventsController extends AppController
            $conditions['includeAttachments'] = true;
        }
        if (isset($this->params['named']['deleted'])) {
-            $conditions['deleted'] = $this->params['named']['deleted'] == 2 ? 0 : 1;
+            $conditions['deleted'] = $this->params['named']['deleted'] == 2 ? 0 : [0, 1];
        }
        if (isset($this->params['named']['toIDS']) && $this->params['named']['toIDS'] != 0) {
            $conditions['to_ids'] = $this->params['named']['toIDS'] == 2 ? 0 : 1;
@ -3284,9 +3284,9 @@ class EventsController extends AppController
        $sgReferenceOnly = false
    ) {
        $paramArray = array(
-            'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
+            'value', 'type', 'category', 'object_relation', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
            'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'returnFormat',
-            'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy'
+            'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'deleted'
        );
        $filterData = array(
            'request' => $this->request,
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@ -390,7 +390,7 @@ class Attribute extends AppModel
        'text' => array('txt', 'TextExport', 'txt'),
        'yara' => array('txt', 'YaraExport', 'yara'),
        'yara-json' => array('json', 'YaraExport', 'json'),
-        'rpz' => array('rpz', 'RPZExport', 'rpz'),
+        'rpz' => array('txt', 'RPZExport', 'rpz'),
        'csv' => array('csv', 'CsvExport', 'csv'),
        'cache' => array('txt', 'CacheExport', 'cache'),
        'attack' => array('html', 'AttackExport', 'html')
@ -2219,9 +2219,8 @@ class Attribute extends AppModel
        }
        $tag = ClassRegistry::init('Tag');
        $params['tags'] = $this->dissectArgs($params['tags']);
-        $tagArray = $tag->fetchTagIds($params['tags'][0], $params['tags'][1]);
-        if (!empty($params['tags'][0]) && empty($tagArray[0]) && empty($params['lax_tags'])) {
-            $tagArray[0] = array(-1);
+        foreach (array(0, 1, 2) as $tag_operator) {
+            $tagArray[$tag_operator] = $tag->fetchTagIdsSimple($params['tags'][$tag_operator]);
        }
        $temp = array();
        if (!empty($tagArray[0])) {
@ -2286,6 +2285,45 @@ class Attribute extends AppModel
                $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field, 1));
            }
        }
+        $temp = array();
+        if (!empty($tagArray[2])) {
+            if ($tagArray[2][0] === -1) {
+                $conditions[] = array('Event.id' => -1);
+            } else {
+                foreach ($tagArray[2] as $k => $anded_tag) {
+                    $subquery_options = array(
+                        'conditions' => array(
+                            'tag_id' => $anded_tag
+                        ),
+                        'fields' => array(
+                            'event_id'
+                        )
+                    );
+                    $lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
+                    $temp[$k]['OR'] = array();
+                    $temp[$k]['OR'] = array_merge(
+                        $temp[$k]['OR'],
+                        $this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field)
+                    );
+                    $subquery_options = array(
+                        'conditions' => array(
+                            'tag_id' => $anded_tag
+                        ),
+                        'fields' => array(
+                            $options['scope'] === 'Event' ? 'Event.id' : 'attribute_id'
+                        )
+                    );
+                    $lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
+                    $temp[$k]['OR'] = array_merge(
+                        $temp[$k]['OR'],
+                        $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field)
+                    );
+                }
+            }
+        }
+        if (!empty($temp)) {
+            $conditions['AND'][] = array('AND' => $temp);
+        }
        $params['tags'] = array();
        if (!empty($tagArray[0]) && empty($options['pop'])) {
            $params['tags']['OR'] = $tagArray[0];
@ -2293,6 +2331,9 @@ class Attribute extends AppModel
        if (!empty($tagArray[1])) {
            $params['tags']['NOT'] = $tagArray[1];
        }
+        if (!empty($tagArray[2]) && empty($options['pop'])) {
+            $params['tags']['AND'] = $tagArray[2];
+        }
        if (empty($params['tags'])) {
            unset($params['tags']);
        }
@ -2604,12 +2645,12 @@ class Attribute extends AppModel
    public function dissectArgs($args)
    {
        if (empty($args)) {
-            return array(0 => array(), 1 => array());
+            return array(0 => array(), 1 => array(), 2 => array());
        }
        if (!is_array($args)) {
            $args = explode('&&', $args);
        }
-        $result = array(0 => array(), 1 => array());
+        $result = array(0 => array(), 1 => array(), 2 => array());
        if (isset($args['OR']) || isset($args['NOT']) || isset($args['AND'])) {
            if (!empty($args['OR'])) {
                $result[0] = $args['OR'];
@ -2617,6 +2658,9 @@ class Attribute extends AppModel
            if (!empty($args['NOT'])) {
                $result[1] = $args['NOT'];
            }
+            if (!empty($args['AND'])) {
+                $result[2] = $args['AND'];
+            }
        } else {
            foreach ($args as $arg) {
                if (substr($arg, 0, 1) == '!') {
@ -3017,6 +3061,11 @@ class Attribute extends AppModel
        }
        if (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
            $params['conditions']['AND']['(Attribute.deleted + 0)'] = 0;
+        } else {
+            if ($options['deleted'] === "only") {
+                $options['deleted'] = 1;
+            }
+            $params['conditions']['AND']['(Attribute.deleted + 0)'] = $options['deleted'];
        }
        if (isset($options['group'])) {
            $params['group'] = empty($options['group']) ? $options['group'] : false;
@ -3892,6 +3941,7 @@ class Attribute extends AppModel
                    'value' => array('function' => 'set_filter_value'),
                    'category' => array('function' => 'set_filter_simple_attribute'),
                    'type' => array('function' => 'set_filter_simple_attribute'),
+                    'object_relation' => array('function' => 'set_filter_simple_attribute'),
                    'tags' => array('function' => 'set_filter_tags', 'pop' => true),
                    'uuid' => array('function' => 'set_filter_uuid'),
                    'deleted' => array('function' => 'set_filter_deleted'),
@ -3993,12 +4043,8 @@ class Attribute extends AppModel
        if (isset($filters['page'])) {
            $params['page'] = $filters['page'];
        }
-        if (!empty($filtes['deleted'])) {
-            $params['deleted'] = 1;
-            if ($params['deleted'] === 'only') {
-                $params['conditions']['AND'][] = array('Attribute.deleted' => 1);
-                $params['conditions']['AND'][] = array('Object.deleted' => 1);
-            }
+        if (!empty($filters['deleted'])) {
+            $params['deleted'] = $filters['deleted'];
        }
        if ($paramsOnly) {
            return $params;
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@ -171,7 +171,7 @@ class Event extends AppModel
        'xml' => array('xml', 'XmlExport', 'xml'),
        'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
        'snort' => array('txt', 'NidsSnortExport', 'rules'),
-        'rpz' => array('rpz', 'RPZExport', 'rpz'),
+        'rpz' => array('txt', 'RPZExport', 'rpz'),
        'text' => array('text', 'TextExport', 'txt'),
        'csv' => array('csv', 'CsvExport', 'csv'),
        'stix' => array('xml', 'Stix1Export', 'xml'),
@ -1606,6 +1606,7 @@ class Event extends AppModel
                    'value' => array('function' => 'set_filter_value'),
                    'category' => array('function' => 'set_filter_simple_attribute'),
                    'type' => array('function' => 'set_filter_simple_attribute'),
+                    'object_relation' => array('function' => 'set_filter_simple_attribute'),
                    'tags' => array('function' => 'set_filter_tags', 'pop' => true),
                    'ignore' => array('function' => 'set_filter_ignore'),
                    'uuid' => array('function' => 'set_filter_uuid'),
@ -1918,20 +1919,47 @@ class Event extends AppModel
            $conditionsAttributes['AND'][] = array('Attribute.to_ids' => 1);
        }
        $softDeletables = array('Attribute', 'Object', 'ObjectReference');
-        if (isset($options['deleted']) && $options['deleted']) {
+        if (isset($options['deleted'])) {
+            if (!is_array($options['deleted'])) {
+                $options['deleted'] = array($options['deleted']);
+            }
+            foreach ($options['deleted'] as $deleted_key => $deleted_value) {
+                if ($deleted_value === 'only') {
+                    $deleted_value = 1;
+                }
+                $options['deleted'][$deleted_key] = intval($deleted_value);
+            }
            if (!$user['Role']['perm_sync']) {
                foreach ($softDeletables as $softDeletable) {
+                    if (in_array(0, $options['deleted'])) {
+                        $deletion_subconditions = array(
+                            sprintf('%s.deleted', $softDeletable) => 0
+                        );
+                    } else {
+                        $deletion_subconditions = array(
+                            '1=0'
+                        );
+                    }
                    ${'conditions' . $softDeletable . 's'}['AND'][] = array(
                        'OR' => array(
-                            '(SELECT events.org_id FROM events WHERE events.id = ' . $softDeletable . '.event_id)' => $user['org_id'],
-                            $softDeletable . '.deleted LIKE' => 0
+                            'AND' => array(
+                                sprintf('(SELECT events.org_id FROM events WHERE events.id = %s.event_id)', $softDeletable) => $user['org_id'],
+                                sprintf('%s.deleted', $softDeletable) => $options['deleted']
+                            ),
+                            $deletion_subconditions
                        )
                    );
                }
+            } else {
+                foreach ($softDeletables as $softDeletable) {
+                    ${'conditions' . $softDeletable . 's'}['AND'][] = array(
+                        sprintf('%s.deleted', $softDeletable) => $options['deleted']
+                    );
+                }
            }
        } else {
            foreach ($softDeletables as $softDeletable) {
-                ${'conditions' . $softDeletable . 's'}['AND'][$softDeletable . '.deleted LIKE'] = 0;
+                ${'conditions' . $softDeletable . 's'}['AND'][$softDeletable . '.deleted'] = 0;
            }
        }
        $proposal_conditions = array('OR' => array('ShadowAttribute.deleted' => 0));
@ -2618,6 +2646,7 @@ class Event extends AppModel
            'tags' => array('function' => 'set_filter_tags'),
            'category' => array('function' => 'set_filter_simple_attribute'),
            'type' => array('function' => 'set_filter_simple_attribute'),
+            'object_relation' => array('function' => 'set_filter_simple_attribute'),
            'from' => array('function' => 'set_filter_timestamp'),
            'to' => array('function' => 'set_filter_timestamp'),
            'last' => array('function' => 'set_filter_timestamp'),
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@ -4682,7 +4682,11 @@ class Server extends AppModel
            $submodule_name=end($submodule_name);
            $submoduleRemote=exec('cd ' . $path . '; git config --get remote.origin.url');
            exec(sprintf('cd %s; git rev-parse HEAD', $path), $submodule_current_commit_id);
-            $submodule_current_commit_id = $submodule_current_commit_id[0];
+            if (!empty($submodule_current_commit_id[0])) {
+                $submodule_current_commit_id = $submodule_current_commit_id[0];
+            } else {
+                $submodule_current_commit_id = null;
+            }
            $status = array(
                'moduleName' => $submodule_name,
                'current' => $submodule_current_commit_id,
--- a/app/Model/Tag.php
+++ b/app/Model/Tag.php
@ -195,6 +195,19 @@ class Tag extends AppModel
        return array($acceptIds, $rejectIds);
    }

+    // find all of the tag Ids that belong to the accepted tags and the rejected tags
+    public function fetchTagIdsSimple($tags = array())
+    {
+        $results = array();
+        if (!empty($tags)) {
+            $results = $this->findTagIdsByTagNames($tags);
+            if (empty($results)) {
+                $results[] = -1;
+            }
+        }
+        return $results;
+    }
+
    // find all of the tag Ids that belong to the accepted tags and the rejected tags
    public function fetchTagIds($accept = array(), $reject = array())
    {
--- a/app/View/Servers/ondemand_action.ctp
+++ b/app/View/Servers/ondemand_action.ctp
@ -65,7 +65,7 @@ $disabledBtnText = $updateLocked ? 'title="' . __('An action is already in progr
    <?php endforeach; ?>
 </div>
 <?php
-    echo $this->element('side_menu', array('menuList' => 'admin', 'menuItem' => 'adminTools'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'admin', 'menuItem' => 'adminTools'));
 ?>

 <script type="text/javascript">
--- a/app/files/feed-metadata/defaults.json
+++ b/app/files/feed-metadata/defaults.json
@ -298,7 +298,7 @@
      "event_id": "5655",
      "publish": false,
      "override_ids": false,
-      "settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\\\/^http:\\\\\\\/\\\\\\\/www.phishtank.com\\\/i\"}}",
+      "settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\\/^http:\\\\\\/\\\\\\/www.phishtank.com\\/i\"}}",
      "input_source": "network",
      "delete_local_file": false,
      "lookup_visible": false,
@ -1519,54 +1519,82 @@
  },
  {
    "Feed": {
-        "id": "77",
-        "name": "malsilo.url",
-        "provider": "MalSilo",
-        "url": "https:\/\/malsilo.gitlab.io\/feeds\/dumps\/url_list.txt",
-        "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
-        "enabled": false,
-        "distribution": "3",
-        "sharing_group_id": "0",
-        "tag_id": "0",
-        "default": false,
-        "source_format": "csv",
-        "fixed_event": true,
-        "delta_merge": false,
-        "event_id": "0",
-        "publish": false,
-        "override_ids": false,
-        "settings": "{\"csv\":{\"value\":\"3\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
-        "input_source": "network",
-        "delete_local_file": false,
-        "lookup_visible": true,
-        "headers": "",
-        "caching_enabled": true
+      "id": "77",
+      "name": "malsilo.url",
+      "provider": "MalSilo",
+      "url": "https://malsilo.gitlab.io/feeds/dumps/url_list.txt",
+      "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
+      "enabled": false,
+      "distribution": "3",
+      "sharing_group_id": "0",
+      "tag_id": "0",
+      "default": false,
+      "source_format": "csv",
+      "fixed_event": true,
+      "delta_merge": false,
+      "event_id": "0",
+      "publish": false,
+      "override_ids": false,
+      "settings": "{\"csv\":{\"value\":\"3\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
+      "input_source": "network",
+      "delete_local_file": false,
+      "lookup_visible": true,
+      "headers": "",
+      "caching_enabled": true
    }
  },
  {
    "Feed": {
-        "id": "78",
-        "name": "malsilo.ipv4",
-        "provider": "MalSilo",
-        "url": "https:\/\/malsilo.gitlab.io\/feeds\/dumps\/ip_list.txt",
-        "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
-        "enabled": false,
-        "distribution": "3",
-        "sharing_group_id": "0",
-        "tag_id": "0",
-        "default": false,
-        "source_format": "csv",
-        "fixed_event": true,
-        "delta_merge": false,
-        "event_id": "0",
-        "publish": false,
-        "override_ids": false,
-        "settings": "{\"csv\":{\"value\":\"3\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
-        "input_source": "network",
-        "delete_local_file": false,
-        "lookup_visible": true,
-        "headers": "",
-        "caching_enabled": true
+      "id": "78",
+      "name": "malsilo.ipv4",
+      "provider": "MalSilo",
+      "url": "https://malsilo.gitlab.io/feeds/dumps/ip_list.txt",
+      "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
+      "enabled": false,
+      "distribution": "3",
+      "sharing_group_id": "0",
+      "tag_id": "0",
+      "default": false,
+      "source_format": "csv",
+      "fixed_event": true,
+      "delta_merge": false,
+      "event_id": "0",
+      "publish": false,
+      "override_ids": false,
+      "settings": "{\"csv\":{\"value\":\"3\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
+      "input_source": "network",
+      "delete_local_file": false,
+      "lookup_visible": true,
+      "headers": "",
+      "caching_enabled": true
    }
-}
+  },
+  {
+    "Feed": {
+      "id": "87",
+      "name": "malshare.com - current all",
+      "provider": "malshare.com",
+      "url": "https://malshare.com/daily/malshare.current.all.txt",
+      "rules": "",
+      "enabled": false,
+      "distribution": "3",
+      "sharing_group_id": "0",
+      "tag_id": "0",
+      "default": false,
+      "source_format": "freetext",
+      "fixed_event": true,
+      "delta_merge": false,
+      "event_id": "0",
+      "publish": false,
+      "override_ids": false,
+      "settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
+      "input_source": "network",
+      "delete_local_file": false,
+      "lookup_visible": true,
+      "headers": "",
+      "caching_enabled": true,
+      "force_to_ids": false,
+      "cached_elements": 3936
+    }
+  }
 ]