MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/2.4' into tools

pull/5843/head
Steve Clement 2020-04-30 19:45:12 +09:00
parent 5a0c500190 f278407e91
commit 32a789a4c5
No known key found for this signature in database
GPG Key ID: 69A20F509BE4AEE9
26 changed files with 531 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
INSTALL/INSTALL.sh
View File

@ -377,18 +377,19 @@ EOF
checkInstaller () {
# Workaround: shasum is not available on RHEL, only checking sha512
if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
chsum=$(cat /tmp/INSTALL.sh.sha512)
if [[ "${chsum}" == "${INSTsum}" ]]; then
echo "SHA512 matches"
else
echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}"
# exit 1 # uncomment when/if PR is merged
fi
if [[ "${chsum}" == "${INSTsum}" ]]; then
echo "SHA512 matches"
else
echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}"
# exit 1 # uncomment when/if PR is merged
fi
else
# TODO: Implement $FLAVOUR checks and install depending on the platform we are on
if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then
sudo apt update
sudo apt install libdigest-sha-perl -qyy
fi
# SHAsums to be computed, not the -- notatiation is for ease of use with rhash
@ -701,12 +702,20 @@ setBaseURL () {
MISP_BASEURL="https://misp.local"
# Webserver configuration
FQDN='misp.local'
else
elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then
MISP_BASEURL='https://localhost:8443'
IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1)
sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443
# Webserver configuration
FQDN='localhost.localdomain'
elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
else
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
fi
}
@ -2651,16 +2660,16 @@ installSupported () {
if [[ "$1" =~ ^PHP= ]]; then
PHP_VER=$(echo $1 |cut -f2 -d=)
if [[ "$PHP_VER" == "7.2" ]]; then
if [[ "$PHP_VER" == 7.2 ]]; then
# Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72
elif [[ "$PHP_VER" == "7.3" ]]; then
elif [[ "$PHP_VER" == 7.3 ]]; then
# Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74
elif [[ "$PHP_VER" == "7.4" ]]; then
elif [[ "$PHP_VER" == 7.4 ]]; then
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73
elif [[ "$PHP_VER" == "7.0" ]]; then
elif [[ "$PHP_VER" == 7.0 ]]; then
# Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp70
fi
@ -3173,6 +3182,7 @@ x86_64-fedora-30
x86_64-debian-stretch
x86_64-debian-buster
x86_64-ubuntu-bionic
x86_64-ubuntu-focal
x86_64-kali-2019.1
x86_64-kali-2019.2
x86_64-kali-2019.3
@ -3187,6 +3197,7 @@ armv7l-debian-jessie
armv7l-debian-stretch
armv7l-debian-buster
armv7l-ubuntu-bionic
armv7l-ubuntu-focal
"
# Check if we actually support this configuration
@ -3208,12 +3219,18 @@ if [ "${FLAVOUR}" == "ubuntu" ]; then
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
fi
if [ "${RELEASE}" == "20.04" ]; then
echo "Install on Ubuntu 20.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported PHP="7.4" && exit || exit
fi
if [ "${RELEASE}" == "18.10" ]; then
echo "Install on Ubuntu 18.10 partially supported, bye."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
fi
if [ "${RELEASE}" == "19.04" ]; then
echo "Install on Ubuntu 19.04 under development."
echo "Install on Ubuntu 19.04 partially supported bye."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
exit 1

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2020-04-27 at 19:39.56
; Generated by RHash v1.3.8 on 2020-04-30 at 15:20.13
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 130366 19:39.56 2020-04-27 INSTALL.sh
INSTALL.sh 8AFDDFA23C1154790947FA1C09DBC7599614F48D 1DC92AFF146065ECB85D5C5C211E252D5C54D6F86A61C29DDED37A9ECA4540E4 3F2C936AEE2773A29DD02477E463402945576DEC32C3003BFF3132B783E2C93D14EA99E8FADFEEE655C746D4C634BE08 DBBF519D97372DDD5337C1F58252BFE9A86B0C8ABB95A375420D8D543C5239D71C24287A75FAFC389DBFDC657B1B4080530D0D3CB44D5F94152F2ABBB9B23174
; 131010 15:20.13 2020-04-30 INSTALL.sh
INSTALL.sh 660E0D51D88B57CE5BE725117482207E39371038 DCF69118CD37B43C308FD25E6BADAF03549BAF0FFA2AC11A1E919005D700F4AC 74E03A8054AF2E4BCB90426A3B813F57BF032734AB7B4E9D4F6F96961D7371FB051180BEE8357642EB9CC58603C13DA3 C4D1D02980808A92E8E11C72A49AA354DDEFA71C6E85FAC739645CEDEB4B36415243F7FB4B8BC75B6AE7B5D9660E0F88A35E884EBD51EA107128B0D7FB20C946

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
8afddfa23c1154790947fa1c09dbc7599614f48d INSTALL.sh
660e0d51d88b57ce5be725117482207e39371038 INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
1dc92aff146065ecb85d5c5c211e252d5c54d6f86a61c29dded37a9eca4540e4 INSTALL.sh
dcf69118cd37b43c308fd25e6badaf03549baf0ffa2ac11a1e919005d700f4ac INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
3f2c936aee2773a29dd02477e463402945576dec32c3003bff3132b783e2c93d14ea99e8fadfeee655c746d4c634be08 INSTALL.sh
74e03a8054af2e4bcb90426a3b813f57bf032734ab7b4e9d4f6f96961d7371fb051180bee8357642eb9cc58603c13da3 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
dbbf519d97372ddd5337c1f58252bfe9a86b0c8abb95a375420d8d543c5239d71c24287a75fafc389dbfdc657b1b4080530d0d3cb44d5f94152f2abbb9b23174 INSTALL.sh
c4d1d02980808a92e8e11c72a49aa354ddefa71c6e85fac739645cedeb4b36415243f7fb4b8bc75b6ae7b5d9660e0f88a35e884ebd51ea107128b0d7fb20c946 INSTALL.sh

12
INSTALL/INSTALL.tpl.sh
View File

@ -278,16 +278,16 @@ installSupported () {
if [[ "$1" =~ ^PHP= ]]; then
PHP_VER=$(echo $1 |cut -f2 -d=)
if [[ "$PHP_VER" == "7.2" ]]; then
if [[ "$PHP_VER" == 7.2 ]]; then
# Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72
elif [[ "$PHP_VER" == "7.3" ]]; then
# Install PHP 7.3 Dependencies - functionLocation('INSTALL.ubuntu2004.md')
elif [[ "$PHP_VER" == 7.3 ]]; then
# Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73
elif [[ "$PHP_VER" == "7.4" ]]; then
# Install PHP 7.4 Dependencies - functionLocation('generic/supportFunctions.md')
elif [[ "$PHP_VER" == 7.4 ]]; then
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74
elif [[ "$PHP_VER" == "7.0" ]]; then
elif [[ "$PHP_VER" == 7.0 ]]; then
# Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp70
fi

2
PyMISP

@ -1 +1 @@
Subproject commit 0faa75824f4dbac2b14919bb17e9d0fef79026d7
Subproject commit 5cc7a1ad57c2e5a90092644e61c3de1e3e449a36

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":124}
{"major":2, "minor":4, "hotfix":125}

4
app/Controller/AppController.php
View File

@ -46,8 +46,8 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
private $__queryVersion = '104';
public $pyMispVersion = '2.4.123';
private $__queryVersion = '105';
public $pyMispVersion = '2.4.125';
public $phpmin = '7.2';
public $phprec = '7.4';
public $pythonmin = '3.6';

12
app/Controller/AttributesController.php
View File

@ -151,6 +151,12 @@ class AttributesController extends AppController
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
if ($this->request->data['Attribute']['distribution'] == 4) {
$sg = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']);
if (empty($sg)) {
throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.'));
}
}
//
// multiple attributes in batch import
//
@ -831,6 +837,12 @@ class AttributesController extends AppController
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
if ($this->request->data['Attribute']['distribution'] == 4) {
$sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']);
if (empty($sg)) {
throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.'));
}
}
$existingAttribute = $this->Attribute->findByUuid($this->Attribute->data['Attribute']['uuid']);
// check if the attribute has a timestamp already set (from a previous instance that is trying to edit via synchronisation)
// check which attribute is newer

10
app/Controller/EventsController.php
View File

@ -3834,7 +3834,7 @@ class EventsController extends AppController
$this->Event->insertLock($this->Auth->user(), $id);
$attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
$default_comment = $this->request->data['Attribute']['default_comment'];
$force = $this->request->data['Attribute']['force'];
$force = $this->_isSiteAdmin() && $this->request->data['Attribute']['force'];
$flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $force);
$this->Flash->info($flashMessage);
$this->redirect(array('controller' => 'events', 'action' => 'view', $id));
@ -4529,16 +4529,20 @@ class EventsController extends AppController
if (!in_array($type, $validTools)) {
throw new MethodNotAllowedException('Invalid type.');
}
App::uses('EventTimelineTool', 'Tools');
$grapher = new EventTimelineTool();
$data = $this->request->is('post') ? $this->request->data : array();
$dataFiltering = array_key_exists('filtering', $data) ? $data['filtering'] : array();
$scope = isset($data['scope']) ? $data['scope'] : 'seen';
$extended = isset($this->params['named']['extended']) ? 1 : 0;
$grapher->construct($this->Event, $this->Auth->user(), $dataFiltering, $extended);
$json = $grapher->get_timeline($id);
if ($scope == 'seen') {
$json = $grapher->get_timeline($id);
} elseif ($scope == 'sightings') {
$json = $grapher->get_sighting_timeline($id);
}
array_walk_recursive($json, function (&$item, $key) {
if (!mb_detect_encoding($item, 'utf-8', true)) {

141
app/Lib/Tools/EventTimelineTool.php
View File

@ -135,4 +135,145 @@
return $this->__json;
}
/*
* Extrapolation strategy:
* - If only positive sightings: Will be from first to last sighting
* - If both positive and false positive: False positive get priority. It will be marked as false positive until next positive sighting
*/
public function get_sighting_timeline($id)
{
$event = $this->__eventModel->fetchEvent($this->__user, array(
'eventid' => $id,
'flatten' => 1,
'includeTagRelations' => 1,
'extended' => $this->__extended_view
));
$this->__json['items'] = array();
if (empty($event)) {
return $this->__json;
} else {
$event = $event[0];
}
$lookupAttribute = array();
foreach ($event['Attribute'] as $k => $attribute) {
$lookupAttribute[$attribute['id']] = &$event['Attribute'][$k];
}
// regroup sightings per attribute
$regroupedSightings = array();
foreach ($event['Sighting'] as $k => $sighting) {
$event['Sighting'][$k]['date_sighting'] *= 1000; // adapt to use micro
$regroupedSightings[$sighting['attribute_id']][] = &$event['Sighting'][$k];
}
// make sure sightings are ordered
uksort($regroupedSightings, function ($a, $b) {
return $a['date_sighting'] > $b['date_sighting'];
});
// generate extrapolation
$now = time()*1000;
foreach ($regroupedSightings as $attributeId => $sightings) {
$i = 0;
while ($i < count($sightings)) {
$sighting = $sightings[$i];
$attribute = $lookupAttribute[$attributeId];
$fpSightingIndex = $this->getNextFalsePositiveSightingIndex($sightings, $i+1);
if ($fpSightingIndex === false) { // No next FP, extrapolate to now
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_positive',
'timestamp' => $attribute['timestamp'],
'first_seen' => $sighting['date_sighting'],
'last_seen' => $now,
);
break;
} else {
// set up until last positive
$pSightingIndex = $fpSightingIndex - 1;
$halfTime = 0;
if ($pSightingIndex == $i) {
// we have only one positive sighting, thus the UP time should be take from a pooling frequence
// for now, consider it UP only for half the time until the next FP
$halfTime = ($sightings[$i+1]['date_sighting'] - $sighting['date_sighting'])/2;
}
$pSighting = $sightings[$pSightingIndex];
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_positive',
'timestamp' => $attribute['timestamp'],
'first_seen' => $sighting['date_sighting'],
'last_seen' => $pSighting['date_sighting'] + $halfTime,
);
// No next FP, extrapolate to now
$fpSighting = $sightings[$fpSightingIndex];
$secondNextPSightingIndex = $this->getNextPositiveSightingIndex($sightings, $fpSightingIndex+1);
if ($secondNextPSightingIndex === false) { // No next P, extrapolate to now
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_negative',
'timestamp' => $attribute['timestamp'],
'first_seen' => $pSighting['date_sighting'] - $halfTime,
'last_seen' => $now,
);
break;
} else {
if ($halfTime > 0) { // We need to fake a previous P
$pSightingIndex = $pSightingIndex+1;
$pSighting = $sightings[$pSightingIndex];
}
// set down until next postive
$secondNextPSighting = $sightings[$secondNextPSightingIndex];
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $pSighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_negative',
'timestamp' => $attribute['timestamp'],
'first_seen' => $pSighting['date_sighting'] - $halfTime,
'last_seen' => $secondNextPSighting['date_sighting'],
);
$i = $secondNextPSightingIndex;
}
}
}
}
return $this->__json;
}
private function getNextFalsePositiveSightingIndex($sightings, $startIndex)
{
for ($i=$startIndex; $i < count($sightings) ; $i++) {
$sighting = $sightings[$i];
if ($sighting['type'] == 1) { // is false positive
return $i;
}
}
return false;
}
private function getNextPositiveSightingIndex($sightings, $startIndex)
{
for ($i=$startIndex; $i < count($sightings) ; $i++) {
$sighting = $sightings[$i];
if ($sighting['type'] == 0) { // is false positive
return $i;
}
}
return false;
}
}

25
app/Model/AppModel.php
View File

@ -78,7 +78,7 @@ class AppModel extends Model
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
51 => false, 52 => false
51 => false, 52 => false, 53 => false
);
public $advanced_updates_description = array(
@ -1383,6 +1383,12 @@ class AppModel extends Model
$this->__addIndex('admin_settings', 'setting');
}
break;
case 53:
if (!empty($this->query("SHOW COLUMNS FROM `user_settings` LIKE 'key';"))) {
$sqlArray[] = "ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
$this->__addIndex('user_settings', 'setting');
}
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2858,4 +2864,21 @@ class AppModel extends Model
return $this->log($message, $type);
}
/**
* Generates random file name in tmp dir.
* @return string
*/
protected function tempFileName()
{
return $this->tempDir() . DS . $this->generateRandomFileName();
}
/**
* @return string
*/
protected function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
}

6
app/Model/Attribute.php
View File

@ -699,7 +699,7 @@ class Attribute extends AppModel
* Only recorrelate if:
* - We are dealing with a new attribute OR
* - The existing attribute's previous state is known AND
* value, type or disable correlation have changed
* value, type, disable correlation or distribution have changed
* This will avoid recorrelations when it's not really needed, such as adding a tag
*/
if (!$created) {
@ -707,7 +707,9 @@ class Attribute extends AppModel
empty($this->old) ||
$this->data['Attribute']['value'] != $this->old['Attribute']['value'] ||
$this->data['Attribute']['disable_correlation'] != $this->old['Attribute']['disable_correlation'] ||
$this->data['Attribute']['type'] != $this->old['Attribute']['type']
$this->data['Attribute']['type'] != $this->old['Attribute']['type'] ||
$this->data['Attribute']['distribution'] != $this->old['Attribute']['distribution'] ||
$this->data['Attribute']['sharing_group_id'] != $this->old['Attribute']['sharing_group_id']
) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);

109
app/Model/Event.php
View File

@ -641,6 +641,12 @@ class Event extends AppModel
if (isset($this->data['Event']['info'])) {
$this->Correlation->updateAll(array('Correlation.info' => $db->value($this->data['Event']['info'])), array('Correlation.event_id' => intval($this->data['Event']['id'])));
}
if (isset($this->data['Event']['distribution'])) {
$this->Correlation->updateAll(array('Correlation.distribution' => $db->value($this->data['Event']['distribution'])), array('Correlation.event_id' => intval($this->data['Event']['id'])));
}
if (isset($this->data['Event']['sharing_group_id'])) {
$this->Correlation->updateAll(array('Correlation.sharing_group_id' => $db->value($this->data['Event']['sharing_group_id'])), array('Correlation.event_id' => intval($this->data['Event']['id'])));
}
}
if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
@ -2150,6 +2156,22 @@ class Event extends AppModel
'Object' => array('name', 'meta-category')
);
foreach ($results as $eventKey => &$event) {
if ($event['Event']['distribution'] == 4 && !in_array($event['Event']['sharing_group_id'], $sgids)) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Event',
'model_id' => $event['Event']['id'],
'email' => $user['email'],
'action' => 'fetchEvent',
'user_id' => $user['id'],
'title' => 'User was able to fetch the event but not the sharing_group it belongs to',
'change' => ''
));
unset($results[$eventKey]); // Current user cannot access sharing_group associated to this event
continue;
}
$this->__attachReferences($user, $event, $sgids, $fields);
$event = $this->Orgc->attachOrgsToEvent($event, $fieldsOrg);
if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) {
@ -2448,7 +2470,11 @@ class Event extends AppModel
}
foreach ($data as $k => $v) {
if ($v['distribution'] == 4) {
$data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup'];
if (isset($sharingGroupData[$v['sharing_group_id']])) {
$data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup'];
} else {
unset($data[$k]); // current user could not fetch the sharing_group
}
}
}
return $data;
@ -3237,10 +3263,10 @@ class Event extends AppModel
return array($bodyevent, $body);
}
private function __captureSGForElement($element, $user)
private function __captureSGForElement($element, $user, $syncLocal=false)
{
if (isset($element['SharingGroup'])) {
$sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user);
$sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user, $syncLocal);
unset($element['SharingGroup']);
} elseif (isset($element['sharing_group_id'])) {
$sg = $this->SharingGroup->checkIfAuthorised($user, $element['sharing_group_id']) ? $element['sharing_group_id'] : false;
@ -3257,17 +3283,17 @@ class Event extends AppModel
// When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know
// or which we need to update. All of that is controlled in this method.
private function __captureObjects($data, $user)
private function __captureObjects($data, $user, $syncLocal=false)
{
// First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it
if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) {
$data['Event'] = $this->__captureSGForElement($data['Event'], $user);
$data['Event'] = $this->__captureSGForElement($data['Event'], $user, $syncLocal);
}
if (!empty($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as $k => $a) {
unset($data['Event']['Attribute']['id']);
if (isset($a['distribution']) && $a['distribution'] == 4) {
$data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user);
$data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user, $syncLocal);
if ($data['Event']['Attribute'][$k] === false) {
unset($data['Event']['Attribute']);
}
@ -3277,7 +3303,7 @@ class Event extends AppModel
if (!empty($data['Event']['Object'])) {
foreach ($data['Event']['Object'] as $k => $o) {
if (isset($o['distribution']) && $o['distribution'] == 4) {
$data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user);
$data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user, $syncLocal);
if ($data['Event']['Object'][$k] === false) {
unset($data['Event']['Object'][$k]);
continue;
@ -3285,7 +3311,7 @@ class Event extends AppModel
}
foreach ($o['Attribute'] as $k2 => $a) {
if (isset($a['distribution']) && $a['distribution'] == 4) {
$data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user);
$data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user, $syncLocal);
if ($data['Event']['Object'][$k]['Attribute'][$k2] === false) {
unset($data['Event']['Object'][$k]['Attribute'][$k2]);
}
@ -3453,6 +3479,24 @@ class Event extends AppModel
return 'blocked';
}
}
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email',
'Server.internal'
)
));
} else {
$server['Server']['internal'] = false;
}
if ($fromXml) {
// Workaround for different structure in XML/array than what CakePHP expects
$data = $this->cleanupEventArrayFromXML($data);
@ -3479,7 +3523,7 @@ class Event extends AppModel
return $existingEvent['Event']['id'];
} else {
if ($fromXml) {
$data = $this->__captureObjects($data, $user);
$data = $this->__captureObjects($data, $user, $server['Server']['internal']);
}
if ($data === false) {
$failedCapture = true;
@ -3487,7 +3531,7 @@ class Event extends AppModel
}
} else {
if ($fromXml) {
$data = $this->__captureObjects($data, $user);
$data = $this->__captureObjects($data, $user, $server['Server']['internal']);
}
if ($data === false) {
$failedCapture = true;
@ -3548,19 +3592,6 @@ class Event extends AppModel
$this->Log = ClassRegistry::init('Log');
if ($saveResult) {
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
)
));
if ($server['Server']['publish_without_email'] == 0) {
$st = "enabled";
} else {
@ -3703,6 +3734,23 @@ class Event extends AppModel
} else {
$existingEvent = $this->findById($id);
}
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
)
));
} else {
$server['Server']['internal'] = false;
}
// If the event exists...
$dateObj = new DateTime();
$date = $dateObj->getTimestamp();
@ -3725,7 +3773,7 @@ class Event extends AppModel
return(array('error' => 'Event could not be saved: Invalid sharing group or you don\'t have access to that sharing group.'));
}
} else {
$data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user);
$data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user, $server['Server']['internal']);
unset($data['Event']['SharingGroup']);
if ($data['Event']['sharing_group_id'] === false) {
return (array('error' => 'Event could not be saved: User not authorised to create the associated sharing group.'));
@ -3846,19 +3894,6 @@ class Event extends AppModel
if ((!empty($data['Event']['published']) && 1 == $data['Event']['published'])) {
// The edited event is from a remote server ?
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
)
));
if ($server['Server']['publish_without_email'] == 0) {
$st = "enabled";
} else {

110
app/Model/Feed.php
View File

@ -247,10 +247,7 @@ class Feed extends AppModel
$data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true);
if (!$isLocal) {
$redis = $this->setupRedis();
if ($redis === false) {
throw new Exception('Could not reach Redis.');
}
$redis = $this->setupRedisWithException();
$redis->del('misp:feed_cache:' . $feed['Feed']['id']);
file_put_contents($feedCache, $data);
}
@ -502,12 +499,17 @@ class Feed extends AppModel
$result = array(
'header' => array(
'Accept' => array('application/json', 'text/plain'),
'Content-Type' => 'application/json',
'MISP-version' => $version,
'MISP-uuid' => Configure::read('MISP.uuid')
'Accept' => array('application/json', 'text/plain'),
'MISP-version' => $version,
'MISP-uuid' => Configure::read('MISP.uuid'),
)
);
// Enable gzipped responses if PHP has 'gzdecode' method
if (function_exists('gzdecode')) {
$result['header']['Accept-Encoding'] = 'gzip';
}
if ($commit) {
$result['header']['commit'] = $commit;
}
@ -1590,24 +1592,51 @@ class Feed extends AppModel
if ($data === false) {
throw new Exception("Could not read local file '$uri'.");
}
return $data;
} else {
throw new Exception("Local file '$uri' doesn't exists.");
}
}
$request = $this->__createFeedRequest($feed['Feed']['headers']);
if ($followRedirect) {
$response = $this->getFollowRedirect($HttpSocket, $uri, $request);
} else {
$request = $this->__createFeedRequest($feed['Feed']['headers']);
$response = $HttpSocket->get($uri, array(), $request);
}
if ($followRedirect) {
$response = $this->getFollowRedirect($HttpSocket, $uri, $request);
} else {
$response = $HttpSocket->get($uri, array(), $request);
}
if ($response === false) {
throw new Exception("Could not reach '$uri'.");
} else if ($response->code != 200) { // intentionally !=
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
}
if ($response === false) {
throw new Exception("Could not reach '$uri'.");
} else if ($response->code != 200) { // intentionally !=
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
$data = $response->body;
$contentEncoding = $response->getHeader('Content-Encoding');
if ($contentEncoding === 'gzip') {
$data = gzdecode($data);
if ($data === false) {
throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed.");
}
} else if ($contentEncoding) {
throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'");
}
$contentType = $response->getHeader('Content-Type');
if ($contentType === 'application/zip') {
$zipFile = new File($this->tempFileName());
$zipFile->write($data);
$zipFile->close();
try {
$data = $this->unzipFirstFile($zipFile);
} catch (Exception $e) {
throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}");
} finally {
$zipFile->delete();
}
$data = $response->body;
}
return $data;
@ -1720,4 +1749,47 @@ class Feed extends AppModel
$this->save($feed);
return $count;
}
/**
* @param File $zipFile
* @return string Uncompressed data
* @throws Exception
*/
private function unzipFirstFile(File $zipFile)
{
if (!class_exists('ZipArchive')) {
throw new Exception("ZIP archive decompressing is not supported.");
}
$zip = new ZipArchive();
$result = $zip->open($zipFile->pwd());
if ($result !== true) {
throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)");
}
if ($zip->numFiles !== 1) {
throw new Exception("Remote server returns ZIP file, that contains multiple files.");
}
$filename = $zip->getNameIndex(0);
if ($filename === false) {
throw new Exception("Remote server returns ZIP file, but there is a problem with reading filename.");
}
$zip->close();
$destinationFile = $this->tempFileName();
$result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
if ($result === false) {
throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted.");
}
$unzipped = new File($destinationFile);
$data = $unzipped->read();
if ($data === false) {
throw new Exception("Couldn't read extracted file content.");
}
$unzipped->delete();
return $data;
}
}

39
app/Model/SharingGroup.php
View File

@ -54,6 +54,10 @@ class SharingGroup extends AppModel
);
private $__sgoCache = array();
private $__sgAuthorisationCache = array(
'save' => array(),
'access' => array()
);
public function beforeValidate($options = array())
@ -353,6 +357,9 @@ class SharingGroup extends AppModel
// returns true if the SG exists and the user is allowed to see it
public function checkIfAuthorised($user, $id, $adminCheck = true)
{
if (isset($this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id])) {
return $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id];
}
if (Validation::uuid($id)) {
$sgid = $this->SharingGroup->find('first', array(
'conditions' => array('SharingGroup.uuid' => $id),
@ -372,8 +379,10 @@ class SharingGroup extends AppModel
return false;
}
if (($adminCheck && $user['Role']['perm_site_admin']) || $this->SharingGroupServer->checkIfAuthorised($id) || $this->SharingGroupOrg->checkIfAuthorised($id, $user['org_id'])) {
$this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = true;
return true;
}
$this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = false;
return false;
}
@ -485,7 +494,7 @@ class SharingGroup extends AppModel
return $results;
}
public function captureSG($sg, $user)
public function captureSG($sg, $user, $syncLocal=false)
{
$existingSG = !isset($sg['uuid']) ? null : $this->find('first', array(
'recursive' => -1,
@ -501,6 +510,34 @@ class SharingGroup extends AppModel
if (!$user['Role']['perm_sharing_group']) {
return false;
}
// check if current user is contained in the SG and we are in a local sync setup
if (!empty($sg['uuid'])) {
if (isset($this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']])) {
$authorisedToSave = $this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']];
} else {
$authorisedToSave = $this->checkIfAuthorisedToSave($user, $sg);
$this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']] = $authorisedToSave;
}
} else {
$authorisedToSave = $this->checkIfAuthorisedToSave($user, $sg);
}
if (!$user['Role']['perm_site_admin'] &&
!($user['Role']['perm_sync'] && $syncLocal ) &&
!$authorisedToSave
) {
$this->Log->create();
$entry = array(
'org' => $user['Organisation']['name'],
'model' => 'SharingGroup',
'model_id' => $sg['SharingGroup']['uuid'],
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Tried to save a sharing group but the user does not belong to it.'
);
$this->Log->save($entry);
return false;
}
$this->create();
$newSG = array();
$attributes = array(

36
app/View/Events/view.ctp
View File

@ -301,27 +301,27 @@
)
)
);
if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) {
$table_data[] = array(
'key' => __('Correlation'),
'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '',
'html' => sprintf(
'%s%s',
$event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'),
(!$mayModify && !$isSiteAdmin) ? '' : sprintf(
}
if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) {
$table_data[] = array(
'key' => __('Correlation'),
'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '',
'html' => sprintf(
'%s%s',
$event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'),
(!$mayModify && !$isSiteAdmin) ? '' : sprintf(
sprintf(
' (<a onClick="getPopup(%s);" style="%scursor:pointer;font-weight:normal;">%s</a>)',
sprintf(
' (<a onClick="getPopup(%s);" style="%scursor:pointer;font-weight:normal;">%s</a>)',
sprintf(
"'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'",
h($event['Event']['id'])
),
$event['Event']['disable_correlation'] ? 'color:white;' : '',
$event['Event']['disable_correlation'] ? __('enable') : __('disable')
)
"'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'",
h($event['Event']['id'])
),
$event['Event']['disable_correlation'] ? 'color:white;' : '',
$event['Event']['disable_correlation'] ? __('enable') : __('disable')
)
)
);
}
)
);
}
?>

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346
Subproject commit cb9bff18e8c94584588ec1333affb4959cdc08d0

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit 3b5451c32518da3e29c575e868d245f27c18dcf4
Subproject commit 84a7bb07a4f1807546cf5c2e03b35dbc0773699d

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 28687d90d575332776480cd5d683361e7485033c
Subproject commit 2c116cbd1ff50b2ed677ea6af08c8a6bf511a6fd

9
app/webroot/css/event-timeline.css
View File

@ -59,6 +59,15 @@
box-shadow: 0 0 20px rgba(82, 168, 236, 1);`
}
.vis-item.sighting_positive {
background-color: green;
border-color: white;
}
.vis-item.sighting_negative {
background-color: red;
border-color: white;
}
.vis-item.object {
background-color: #3465a4;
border-color: black;

59
app/webroot/js/event-timeline.js
View File

@ -29,10 +29,16 @@ var options = {
return build_object_template(item);
case "object_attribute":
console.log('Error');
console.log('Error: Group not valid');
break;
default:
if (item.className == "sighting_positive" || item.className == "sighting_negative") {
return build_sighting_template(item);
} else {
console.log(item)
console.log('Error: Unkown group');
}
break;
}
},
@ -199,6 +205,13 @@ function build_object_template(obj) {
return html;
}
function build_sighting_template(attr){
var span = $('<span data-itemID="'+attr.id+'">');
span.text(attr.content);
var html = span[0].outerHTML;
return html;
}
function contain_seen_attribute(obj) {
if (obj['Attribute'] === undefined) {
return false;
@ -372,6 +385,8 @@ function map_scope(val) {
return 'seen';
case 'Object relationship':
return 'relationship';
case 'Sightings':
return 'sightings';
default:
return 'seen';
}
@ -400,7 +415,8 @@ function update_badge() {
function reload_timeline() {
update_badge();
var payload = {scope: map_scope($('#select_timeline_scope').val())};
var selectedScope = map_scope($('#select_timeline_scope').val());
var payload = {scope: selectedScope};
$.ajax({
url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json",
dataType: 'json',
@ -413,6 +429,8 @@ function reload_timeline() {
},
success: function( data, textStatus, jQxhr ){
items_timeline.clear();
mapping_text_to_id = new Map();
var itemIds = {};
for (var item of data.items) {
item.className = item.group;
item.orig_id = item.id;
@ -420,16 +438,45 @@ function reload_timeline() {
set_spanned_time(item);
if (item.group == 'object') {
for (var attr of item.Attribute) {
mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id);
if (selectedScope == 'sightings') {
var k = attr.contentType+': '+attr.content+' ('+item.orig_id.split('-')[0]+')'
if (!mapping_text_to_id.get(k)) {
mapping_text_to_id.set(k, item.id);
}
} else {
mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id);
}
adjust_text_length(attr);
}
} else {
mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id);
if (selectedScope == 'sightings') {
var k = item.content+' ('+item.orig_id.split('-')[0]+')'
if (!mapping_text_to_id.get(k)) {
mapping_text_to_id.set(k, item.id);
}
} else {
mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id);
}
adjust_text_length(item);
}
itemIds[item.attribute_id] = item.content;
if (selectedScope == 'sightings') {
item.group = item.attribute_id;
item.content = '';
}
}
items_timeline.add(data.items);
handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').prop('checked'), false)
if (selectedScope == 'sightings') {
var groups = Object.keys(itemIds).map(function(id) {
return {id: id, content: itemIds[id]}
})
eventTimeline.setGroups(groups);
eventTimeline.setOptions({selectable: false});
} else {
eventTimeline.setOptions({selectable: true});
eventTimeline.setGroups([]);
}
},
error: function( jqXhr, textStatus, errorThrown ){
console.log( errorThrown );
@ -610,11 +657,11 @@ function init_popover() {
label: "Scope",
tooltip: "The time scope represented by the timeline",
event: function(value) {
if (value == "First seen/Last seen") {
if (value == "First seen/Last seen" || value == "Sightings") {
reload_timeline();
}
},
options: ["First seen/Last seen"],
options: ["First seen/Last seen", "Sightings"],
default: "First seen/Last seen"
});

4
db_schema.json
View File

@ -6567,5 +6567,5 @@
"id"
]
},
"db_version": "51"
}
"db_version": "53"
}

10
docs/generic/supportFunctions.md
View File

@ -541,12 +541,20 @@ setBaseURL () {
MISP_BASEURL="https://misp.local"
# Webserver configuration
FQDN='misp.local'
else
elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then
MISP_BASEURL='https://localhost:8443'
IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1)
sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443
# Webserver configuration
FQDN='localhost.localdomain'
elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
else
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
fi
}