MISP/app/Model/Sighting.php

1114 lines
43 KiB
PHP
Raw Normal View History

<?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
2016-08-21 23:53:56 +02:00
/**
* @property Attribute $Attribute
2020-10-05 14:42:49 +02:00
* @property Event $Event
* @property Organisation $Organisation
*/
class Sighting extends AppModel
{
2020-11-11 09:00:50 +01:00
const ONE_DAY = 86400; // in seconds
// Possible values of `Plugin.Sightings_policy` setting
const SIGHTING_POLICY_EVENT_OWNER = 0,
SIGHTING_POLICY_SIGHTING_REPORTER = 1,
SIGHTING_POLICY_EVERYONE = 2;
private $orgCache = [];
public $useTable = 'sightings';
2016-08-21 23:53:56 +02:00
public $recursive = -1;
public $actsAs = array(
'Containable',
);
public $validate = array(
'event_id' => 'numeric',
'attribute_id' => 'numeric',
'org_id' => 'numeric',
'date_sighting' => 'numeric',
2020-10-05 14:42:49 +02:00
'uuid' => 'uuid',
'type' => array(
'rule' => array('inList', array(0, 1, 2)),
'message' => 'Invalid type. Valid options are: 0 (Sighting), 1 (False-positive), 2 (Expiration).'
)
);
public $belongsTo = array(
'Attribute',
'Event',
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id'
),
);
public $type = array(
0 => 'sighting',
1 => 'false-positive',
2 => 'expiration'
);
public $validFormats = array(
'json' => array('json', 'JsonExport', 'json'),
'xml' => array('xml', 'XmlExport', 'xml'),
'csv' => array('csv', 'CsvExport', 'csv')
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['Sighting']['id']) && empty($this->data['Sighting']['date_sighting'])) {
$this->data['Sighting']['date_sighting'] = date('Y-m-d H:i:s');
}
if (empty($this->data['Sighting']['uuid'])) {
$this->data['Sighting']['uuid'] = CakeText::uuid();
2020-10-05 14:42:49 +02:00
} else {
$this->data['Sighting']['uuid'] = strtolower($this->data['Sighting']['uuid']);
}
return true;
}
public function afterSave($created, $options = array())
{
parent::afterSave($created, $options = array());
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_sighting_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_sighting_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$sighting = $this->getSighting($this->id, $user);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($sighting, 'add');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $sighting, 'add');
}
}
return true;
}
public function beforeDelete($cascade = true)
{
parent::beforeDelete();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_sighting_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_sighting_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$sighting = $this->getSighting($this->id, $user);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($sighting, 'delete');
}
if ($pubToKafka) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $sighting, 'delete');
}
}
}
public function captureSighting($sighting, $attribute_id, $event_id, $user)
{
$org_id = 0;
if (!empty($sighting['Organisation'])) {
$org_id = $this->Organisation->captureOrg($sighting['Organisation'], $user);
}
if (isset($sighting['id'])) {
unset($sighting['id']);
}
$sighting['org_id'] = $org_id;
$sighting['event_id'] = $event_id;
$sighting['attribute_id'] = $attribute_id;
$this->create();
return $this->save($sighting);
}
/**
* @param int $id
* @param array $user
* @param bool $withEvent
* @return array
*/
public function getSighting($id, array $user, $withEvent = true)
{
$sighting = $this->find('first', array(
'recursive' => -1,
'contain' => array(
'Attribute' => array(
'fields' => array('Attribute.value', 'Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids')
),
'Event' => array(
'fields' => $withEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'],
)
),
'conditions' => array('Sighting.id' => $id)
));
if (empty($sighting)) {
return array();
}
$ownEvent = $user['Role']['perm_site_admin'] || $sighting['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
$sightingPolicy = $this->sightingsPolicy();
// if sighting policy == 0 then return false if the sighting doesn't belong to the user
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) {
return array();
}
}
// if sighting policy == 1, the user can only see the sighting if they've sighted something in the event once
else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
2020-11-11 09:00:50 +01:00
if (!$this->isReporter($sighting['Sighting']['event_id'], $user['org_id'])) {
return array();
}
}
}
// Put event organisation name from cache
if ($withEvent) {
$sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name'];
}
$anonymise = Configure::read('Plugin.Sightings_anonymise');
if ($anonymise && $sighting['Sighting']['org_id'] != $user['org_id']) {
unset($sighting['Sighting']['org_id']);
}
// rearrange it to match the event format of fetchevent
$result = array(
'Sighting' => $sighting['Sighting']
);
if ($withEvent) {
$result['Sighting']['Event'] = $sighting['Event'];
}
$result['Sighting']['Attribute'] = $sighting['Attribute'];
return $result;
}
2017-11-03 15:18:46 +01:00
/**
* @param array $tagIds
* @param array $user
* @param null|string $type
* @return array
*/
public function tagsSparkline(array $tagIds, array $user, $type = null)
{
if (empty($tagIds)) {
return [];
}
$conditions = ['Sighting.date_sighting >' => $this->getMaximumRange()];
if ($type !== null) {
$conditions['Sighting.type'] = $type;
}
$sightingsPolicy = $this->sightingsPolicy();
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$conditions['Sighting.org_id'] = $user['org_id'];
}
// TODO: Currently, we dont support `SIGHTING_POLICY_SIGHTING_REPORTER` for tags
$sparklineData = [];
foreach (['event', 'attribute'] as $context) {
$sightings = $this->fetchGroupedSightingsForTags($tagIds, $conditions, $context);
$objectElement = ucfirst($context) . 'Tag';
foreach ($sightings as $sighting) {
$tagId = $sighting[$objectElement]['tag_id'];
$date = $sighting['Sighting']['date'];
$count = (int)$sighting['Sighting']['sighting_count'];
if (isset($sparklineData[$tagId][$date]['sighting'])) {
$sparklineData[$tagId][$date]['sighting'] += $count;
} else {
$sparklineData[$tagId][$date]['sighting'] = $count;
}
}
}
return $this->generateSparkline($sparklineData, false);
}
2020-11-11 09:00:50 +01:00
/**
* @param array $attributes Attribute must contain Event
* @param array $user
* @param bool $csvWithFalsePositive
* @return array[]
*/
public function attributesStatistics(array $attributes, array $user, $csvWithFalsePositive = false)
{
if (empty($attributes)) {
return ['data' => [], 'csv' => []];
}
$sightingsPolicy = $this->sightingsPolicy();
$conditions = [];
foreach ($attributes as $attribute) {
$attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
$ownEvent = $user['Role']['perm_site_admin'] || $attribute['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$attributeConditions['Sighting.org_id'] = $user['org_id'];
} else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) {
continue; // skip attribute
}
}
}
$conditions['OR'][] = $attributeConditions;
}
$groupedSightings = $this->fetchGroupedSightings($conditions, $user);
return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
}
/**
* @param array $events
* @param array $user
* @param bool $csvWithFalsePositive
* @return array
*/
public function eventsStatistic(array $events, array $user, $csvWithFalsePositive = false)
{
if (empty($events)) {
return ['data' => [], 'csv' => []];
}
$sightingPolicy = $this->sightingsPolicy();
$conditions = [];
foreach ($events as $event) {
$eventCondition = ['Sighting.event_id' => $event['Event']['id']];
$ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$eventCondition['Sighting.org_id'] = $user['org_id'];
} else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($event['Event']['id'], $user['org_id'])) {
continue;
}
}
}
$conditions['OR'][] = $eventCondition;
}
$groupedSightings = $this->fetchGroupedSightings($conditions, $user);
return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
}
/**
* @param array $conditions
* @param array $user
* @return array
*/
private function fetchGroupedSightings(array $conditions, array $user)
{
if (empty($conditions)) {
return [];
}
// Returns date in `Y-m-d` format
$this->virtualFields['date'] = $this->dateVirtualColumn();
2020-11-11 09:00:50 +01:00
$this->virtualFields['sighting_count'] = 'COUNT(id)';
$this->virtualFields['last_timestamp'] = 'MAX(date_sighting)';
2020-11-11 09:00:50 +01:00
$groupedSightings = $this->find('all', array(
'conditions' => $conditions,
'fields' => ['org_id', 'attribute_id', 'type', 'date', 'last_timestamp', 'sighting_count'],
2020-11-11 09:00:50 +01:00
'recursive' => -1,
'group' => ['org_id', 'attribute_id', 'type', 'date'],
'order' => ['date'], // from oldest
2020-11-11 09:00:50 +01:00
));
unset(
$this->virtualFields['date'],
$this->virtualFields['sighting_count'],
$this->virtualFields['last_timestamp']
);
2020-11-11 09:00:50 +01:00
return $this->attachOrgToSightings($groupedSightings, $user, false);
}
/**
* @param array $tagIds
* @param array $conditions
* @param string $context
* @return array
*/
private function fetchGroupedSightingsForTags(array $tagIds, array $conditions, $context)
{
$conditions[ucfirst($context) . 'Tag.tag_id'] = $tagIds;
// Temporary bind EventTag or AttributeTag model
$this->bindModel([
'hasOne' => [
ucfirst($context) . 'Tag' => [
'foreignKey' => false,
'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id',
]
]
]);
// Returns date in `Y-m-d` format
$this->virtualFields['date'] = $this->dateVirtualColumn();
$this->virtualFields['sighting_count'] = 'COUNT(Sighting.id)';
$sightings = $this->find('all', [
'recursive' => -1,
'contain' => [ucfirst($context) . 'Tag'],
'conditions' => $conditions,
'fields' => [ucfirst($context) . 'Tag.tag_id', 'date', 'sighting_count'],
'group' => [ucfirst($context) . 'Tag.id', 'date'],
'order' => ['date'], // from oldest
]);
unset($this->virtualFields['date'], $this->virtualFields['sighting_count']);
return $sightings;
}
2020-11-11 09:00:50 +01:00
/**
* @param array $groupedSightings
* @param bool $csvWithFalsePositive
* @return array[]
*/
private function generateStatistics(array $groupedSightings, $csvWithFalsePositive = false)
{
$sightingsData = [];
$sparklineData = [];
$range = $this->getMaximumRange();
foreach ($groupedSightings as $sighting) {
$type = $this->type[$sighting['type']];
$orgName = isset($sighting['Organisation']['name']) ? $sighting['Organisation']['name'] : __('Others');
$count = (int)$sighting['sighting_count'];
$inRange = strtotime($sighting['date']) >= $range;
2020-11-11 09:00:50 +01:00
foreach ([$sighting['attribute_id'], 'all'] as $needle) {
if (!isset($sightingsData[$needle][$type])) {
$sightingsData[$needle][$type] = ['count' => 0, 'orgs' => []];
}
$ref = &$sightingsData[$needle][$type];
$ref['count'] += $count;
if (!isset($ref['orgs'][$orgName])) {
$ref['orgs'][$orgName] = ['count' => $count, 'date' => $sighting['last_timestamp']];
} else {
$ref['orgs'][$orgName]['count'] += $count;
$ref['orgs'][$orgName]['date'] = $sighting['last_timestamp'];
}
if ($inRange) {
if (isset($sparklineData[$needle][$sighting['date']][$type])) {
$sparklineData[$needle][$sighting['date']][$type] += $count;
2020-11-11 09:00:50 +01:00
} else {
$sparklineData[$needle][$sighting['date']][$type] = $count;
2020-11-11 09:00:50 +01:00
}
}
}
}
return ['data' => $sightingsData, 'csv' => $this->generateSparkline($sparklineData, $csvWithFalsePositive)];
}
/**
* @param array $sparklineData
* @param bool $csvWithFalsePositive
* @return array
*/
private function generateSparkline(array $sparklineData, $csvWithFalsePositive)
{
2020-11-11 09:00:50 +01:00
$todayString = date('Y-m-d');
$today = strtotime($todayString);
// If nothing found, generate default "empty" CSV for 'all'
if (!isset($sparklineData['all'])) {
$sparklineData['all'][$todayString] = null;
}
$csv = [];
foreach ($sparklineData as $object => $data) {
$startDate = key($data); // oldest date for sparkline
$startDate = strtotime($startDate) - (self::ONE_DAY * 3);
$csvForObject = $csvWithFalsePositive ? 'Date,Sighting,False-positive\n' : 'Date,Close\n';
for ($date = $startDate; $date <= $today; $date += self::ONE_DAY) {
$dateAsString = date('Y-m-d', $date);
$csvForObject .= $dateAsString . ',' . (isset($data[$dateAsString]['sighting']) ? $data[$dateAsString]['sighting'] : '0');
if ($csvWithFalsePositive) {
$csvForObject .= ',' . (isset($data[$dateAsString]['false-positive']) ? $data[$dateAsString]['false-positive'] : '0');
}
$csvForObject .= '\n';
}
$csv[$object] = $csvForObject;
}
return $csv;
2020-11-11 09:00:50 +01:00
}
/**
* @param array $sightings
* @param array $user
* @param false $forSync
* @return array
*/
private function attachOrgToSightings(array $sightings, array $user, $forSync = false)
{
$showOrg = Configure::read('MISP.showorg');
$anonymise = Configure::read('Plugin.Sightings_anonymise');
$anonymiseAs = Configure::read('Plugin.Sightings_anonymise_as');
$anonOrg = null;
if ($forSync && !empty($anonymiseAs)) {
$anonOrg = $this->getOrganisationById($anonymiseAs);
}
foreach ($sightings as $k => $sighting) {
$sighting = $sighting['Sighting'];
if ($showOrg && $sighting['org_id']) {
$sighting['Organisation'] = $this->getOrganisationById($sighting['org_id']);
}
if ($sighting['org_id'] != $user['org_id'] && ($anonymise || !empty($anonOrg))) {
if (empty($anonOrg)) {
unset($sighting['org_id']);
unset($sighting['Organisation']);
} else {
$sighting['org_id'] = $anonOrg['Organisation']['id'];
$sighting['Organisation'] = $anonOrg['Organisation'];
}
}
$sightings[$k] = $sighting;
}
$this->orgCache = []; // clear org cache
return $sightings;
}
/**
* @param array $event
* @param array $user
* @param array|int|null $attribute Attribute model or attribute ID
* @param array|bool $extraConditions
* @param bool $forSync
* @return array|int
*/
public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false, $forSync = false)
{
$contain = [];
$conditions = array('Sighting.event_id' => $event['Event']['id']);
if (isset($attribute['Attribute']['id'])) {
$conditions['Sighting.attribute_id'] = $attribute['Attribute']['id'];
} elseif (is_numeric($attribute)) {
$conditions['Sighting.attribute_id'] = $attribute;
$attribute = $this->Attribute->find('first', [
'recursive' => -1,
'conditions' => ['Attribute.id' => $attribute],
'fields' => ['Attribute.uuid']
]);
} else {
$contain['Attribute'] = ['fields' => 'Attribute.uuid'];
}
2020-11-11 09:00:50 +01:00
$ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
$sightingPolicy = $this->sightingsPolicy();
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$conditions['Sighting.org_id'] = $user['org_id'];
} elseif ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($event['Event']['id'], $user['org_id'])) {
return array();
}
}
}
if ($extraConditions !== false) {
$conditions['AND'] = $extraConditions;
}
$sightings = $this->find('all', array(
'conditions' => $conditions,
'recursive' => -1,
'contain' => $contain,
));
if (empty($sightings)) {
return array();
}
foreach ($sightings as $k => $sighting) {
if (isset($sighting['Attribute']['uuid'])) {
$sighting['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
} else {
$sighting['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
}
2020-11-11 09:00:50 +01:00
$sightings[$k] = $sighting;
}
2020-11-11 09:00:50 +01:00
return $this->attachOrgToSightings($sightings, $user, $forSync);
}
public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false, $saveOnBehalfOf = false)
{
if (!in_array($type, array(0, 1, 2))) {
return 'Invalid type, please change it before you POST 1000000 sightings.';
}
if ($sighting_uuid) {
// Since sightings are immutable (it is not possible to change it from web interface), we can check
// if sighting with given uuid already exists and quit early
$existing_sighting = $this->find('count', array(
'recursive' => -1,
'conditions' => array('uuid' => $sighting_uuid),
'callbacks' => false,
));
if ($existing_sighting) {
return 0;
}
}
$conditions = array();
if ($id && $id !== 'stix') {
$id = $this->explodeIdList($id);
if (!is_array($id) && strlen($id) == 36) {
$conditions = array('Attribute.uuid' => $id);
} else {
$conditions = array('Attribute.id' => $id);
}
} else {
if (!$values) {
return 'No valid attributes found.';
}
if (!is_array($values)) {
$values = array($values);
}
foreach ($values as $value) {
foreach (array('value1', 'value2') as $field) {
$conditions['OR'][] = array(
'LOWER(Attribute.' . $field . ') LIKE' => strtolower($value)
);
}
}
}
$attributes = $this->Attribute->fetchAttributesSimple($user, [
'conditions' => $conditions,
'fields' => ['Attribute.id', 'Attribute.event_id'],
]);
if (empty($attributes)) {
return 'No valid attributes found that match the criteria.';
}
$sightingsAdded = 0;
foreach ($attributes as $attribute) {
if ($type === '2') {
// remove existing expiration by the same org if it exists
$this->deleteAll(array(
'Sighting.org_id' => $user['org_id'],
'Sighting.type' => $type,
'Sighting.attribute_id' => $attribute['Attribute']['id'],
));
}
$this->create();
$sighting = array(
'attribute_id' => $attribute['Attribute']['id'],
'event_id' => $attribute['Attribute']['event_id'],
'org_id' => ($saveOnBehalfOf === false) ? $user['org_id'] : $saveOnBehalfOf,
'date_sighting' => $timestamp,
'type' => $type,
'source' => $source,
);
// zeroq: allow setting a specific uuid
if ($sighting_uuid) {
$sighting['uuid'] = $sighting_uuid;
}
$result = $this->save($sighting);
if ($result === false) {
return json_encode($this->validationErrors);
}
++$sightingsAdded;
if ($publish) {
$this->Event->publishRouter($sighting['event_id'], null, $user, 'sightings');
}
}
return $sightingsAdded;
}
public function handleStixSighting($data)
{
$randomFileName = $this->generateRandomFileName();
$tempFile = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName, true, 0644);
// save the json_encoded event(s) to the temporary file
if (!$tempFile->write($data)) {
return array('success' => 0, 'message' => 'Could not write the Sightings file to disk.');
}
$tempFile->close();
$scriptFile = APP . "files" . DS . "scripts" . DS . "stixsighting2misp.py";
// Execute the python script and point it to the temporary filename
$result = shell_exec($this->getPythonVersion() . ' ' . $scriptFile . ' ' . $randomFileName);
// The result of the script will be a returned JSON object with 2 variables: success (boolean) and message
// If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along
$result = json_decode($result, true);
if ($result['success'] == 1) {
$file = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName . ".out");
$result['data'] = $file->read();
$file->close();
$file->delete();
}
$tempFile->delete();
return $result;
}
/**
* @return bool
* @deprecated
*/
public function addUuids()
{
$sightings = $this->find('all', array(
'recursive' => -1,
'conditions' => array('uuid' => '')
));
$this->saveMany($sightings);
return true;
}
public function explodeIdList($id)
{
if (strpos($id, '|')) {
$id = explode('|', $id);
foreach ($id as $k => $v) {
if (!is_numeric($v)) {
unset($id[$k]);
}
}
$id = array_values($id);
}
return $id;
}
/**
* @param array $user
* @param $ids
* @param string $context
* @param int|false $orgId
* @param int|false $sightingsType
* @param bool $orderDesc
* @return array|int|null
* @throws Exception
*/
public function listSightings(array $user, $ids, $context, $orgId = false, $sightingsType = false, $orderDesc = true)
{
$this->Event = ClassRegistry::init('Event');
$ids = is_array($ids) ? $ids : $this->explodeIdList($ids);
$objectIds = [];
$eventOwnerOrgIdList = [];
if ($context === 'attribute') {
$objects = $this->Event->Attribute->fetchAttributes($user, ['conditions' => ['Attribute.id' => $ids, 'Attribute.deleted' => 0], 'flatten' => 1]);
foreach ($objects as $object) {
$objectIds[] = $object['Attribute']['id'];
$eventOwnerOrgIdList[$object['Event']['id']] = $object['Event']['orgc_id'];
}
} elseif ($context === 'event') {
// let's set the context to event here, since we reuse the variable later on for some additional lookups.
// Passing $context = 'org' could have interesting results otherwise...
$objects = $this->Event->fetchSimpleEvents($user, ['conditions' => ['Event.id' => $ids]]);
foreach ($objects as $object) {
$objectIds[] = $object['Event']['id'];
$eventOwnerOrgIdList[$object['Event']['id']] = $object['Event']['orgc_id'];
}
} else {
throw new InvalidArgumentException("Invalid context '$context'.");
}
unset($objects);
if (empty($objectIds)) {
throw new MethodNotAllowedException('Invalid object.');
}
$conditions = array(
'Sighting.' . $context . '_id' => $objectIds
);
if ($orgId) {
$conditions[] = array('Sighting.org_id' => $orgId);
}
if ($sightingsType !== false) {
$conditions[] = array('Sighting.type' => $sightingsType);
}
$sightings = $this->find('all', array(
'conditions' => $conditions,
'recursive' => -1,
'contain' => array('Organisation.name'),
'order' => array(sprintf('Sighting.date_sighting %s', $orderDesc ? 'DESC' : ''))
));
if (empty($sightings)) {
return [];
}
if ($user['Role']['perm_site_admin']) {
return $sightings; // site admin can see all sightings, do not limit him
}
$sightingsPolicy = $this->sightingsPolicy();
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$userOrgId = $user['org_id'];
foreach ($sightings as $k => $sighting) {
if ($eventOwnerOrgIdList[$sighting['Sighting']['event_id']] !== $userOrgId && $sighting['Sighting']['org_id'] !== $userOrgId) {
unset($sightings[$k]);
}
}
$sightings = array_values($sightings);
} else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
$eventsWithOwnSightings = array();
foreach ($sightings as $k => $sighting) {
$eventId = $sighting['Sighting']['event_id'];
if (!isset($eventsWithOwnSightings[$eventId])) {
$isReporter = $this->isReporter($eventId, $user['org_id']);
if ($isReporter) {
$eventsWithOwnSightings[$eventId] = true;
} else {
$ownEvent = $eventOwnerOrgIdList[$eventId] == $user['org_id'];
$eventsWithOwnSightings[$eventId] = $ownEvent;
}
}
if (!$eventsWithOwnSightings[$eventId]) {
unset($sightings[$k]);
}
}
$sightings = array_values($sightings);
}
if (Configure::read('Plugin.Sightings_anonymise')) {
foreach ($sightings as $k => $v) {
if ($v['Sighting']['org_id'] != $user['org_id']) {
$sightings[$k]['Organisation']['name'] = '';
$sightings[$k]['Sighting']['org_id'] = 0;
}
}
}
return $sightings;
}
public function restSearch($user, $returnFormat, $filters)
{
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(__('Invalid context.'));
}
// ensure that an id or uuid is provided if context is set
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
App::uses($this->validFormats[$returnFormat][1], 'Export');
$exportTool = new $this->validFormats[$returnFormat][1]();
2018-10-23 13:39:29 +02:00
// construct filtering conditions
if (isset($filters['from']) && isset($filters['to'])) {
$timeCondition = array($filters['from'], $filters['to']);
unset($filters['from']);
unset($filters['to']);
} elseif (isset($filters['last'])) {
$timeCondition = $filters['last'];
unset($filters['last']);
} else {
$timeCondition = '30d';
}
$contain = [];
$conditions = $this->Attribute->setTimestampConditions($timeCondition, [], $scope = 'Sighting.date_sighting');
if (isset($filters['type'])) {
$conditions['Sighting.type'] = $filters['type'];
}
if (isset($filters['org_id'])) {
$this->Organisation = ClassRegistry::init('Organisation');
if (!is_array($filters['org_id'])) {
$filters['org_id'] = array($filters['org_id']);
}
foreach ($filters['org_id'] as $k => $org_id) {
if (Validation::uuid($org_id)) {
$org = $this->Organisation->find('first', array(
'conditions' => array('Organisation.uuid' => $org_id),
'recursive' => -1,
'fields' => array('Organisation.id'),
));
if (empty($org)) {
$filters['org_id'][$k] = -1;
} else {
$filters['org_id'][$k] = $org['Organisation']['id'];
}
}
}
$conditions['Sighting.org_id'] = $filters['org_id'];
}
if (isset($filters['source'])) {
$conditions['Sighting.source'] = $filters['source'];
}
if (!empty($filters['id'])) {
if ($filters['context'] === 'attribute') {
$conditions['Sighting.attribute_id'] = $filters['id'];
} elseif ($filters['context'] === 'event') {
$conditions['Sighting.event_id'] = $filters['id'];
}
}
if (!empty($filters['uuid'])) {
if ($filters['context'] === 'attribute') {
$conditions['Attribute.uuid'] = $filters['uuid'];
$contain[] = 'Attribute';
} elseif ($filters['context'] === 'event') {
$conditions['Event.uuid'] = $filters['uuid'];
$contain[] = 'Event';
}
}
2018-10-23 13:39:29 +02:00
// fetch sightings matching the query
$sightingIds = $this->find('list', array(
'recursive' => -1,
'conditions' => $conditions,
'fields' => array('id'),
'contain' => $contain,
));
$includeAttribute = isset($filters['includeAttribute']) && $filters['includeAttribute'];
$includeEvent = isset($filters['includeEvent']) && $filters['includeEvent'];
$requestedAttributes = ['id', 'attribute_id', 'event_id', 'org_id', 'date_sighting', 'uuid', 'source', 'type'];
if ($includeAttribute) {
$requestedAttributes = array_merge($requestedAttributes, ['attribute_uuid', 'attribute_type', 'attribute_category', 'attribute_to_ids', 'attribute_value']);
}
if ($includeEvent) {
$requestedAttributes = array_merge($requestedAttributes, ['event_uuid', 'event_orgc_id', 'event_org_id', 'event_info', 'event_Orgc_name']);
}
$filters['requested_attributes'] = $requestedAttributes;
$exportToolParams = array(
'user' => $user,
'params' => ['conditions' => []], //result already filtered
'returnFormat' => $returnFormat,
'scope' => 'Sighting',
'filters' => $filters
);
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$separator = $exportTool->separator($exportToolParams);
foreach ($sightingIds as $sightingId) {
// apply ACL and sighting policies
$sighting = $this->getSighting($sightingId, $user, $includeEvent);
if (!empty($sighting)) {
$sighting['Sighting']['value'] = $sighting['Sighting']['Attribute']['value'];
if (!$includeAttribute) {
unset($sighting['Sighting']['Attribute']);
}
$tmpfile->writeWithSeparator($exportTool->handler($sighting, $exportToolParams), $separator);
}
}
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
/**
* @param int|string $eventId Event ID or UUID
* @param array $sightings
* @param array $user
* @param null $passAlong
* @return int Number of saved sightings
* @throws Exception
*/
public function bulkSaveSightings($eventId, array $sightings, array $user, $passAlong = null)
{
$event = $this->Event->fetchSimpleEvent($user, $eventId);
if (empty($event)) {
throw new NotFoundException('Event not found or not accessible by this user.');
}
// Since sightings are immutable (it is not possible to change it from web interface), we can check
// if sighting with given uuid already exists and skip them
$existingSighting = $this->find('list', [
'fields' => ['uuid'],
'recursive' => -1,
'conditions' => ['uuid' => array_column($sightings, 'uuid')],
'callbacks' => false,
]);
// Move UUID to array key
$existingSighting = array_flip($existingSighting);
// Fetch attributes IDs and event IDs
$attributesToTransform = $this->Attribute->fetchAttributesSimple($user, [
'conditions' => ['Attribute.uuid' => array_unique(array_column($sightings, 'attribute_uuid'))],
'fields' => ['Attribute.id', 'Attribute.uuid', 'Attribute.event_id'],
]);
$attributes = [];
foreach ($attributesToTransform as $attribute) {
$attributes[$attribute['Attribute']['uuid']] = [$attribute['Attribute']['id'], $attribute['Attribute']['event_id']];
}
$saved = 0;
foreach ($sightings as $s) {
if (isset($existingSighting[$s['uuid']])) {
continue; // sighting already exists
}
if (!isset($attributes[$s['attribute_uuid']])) {
continue; // attribute doesn't exists or user don't have permission to access it
}
list($attributeId, $eventId) = $attributes[$s['attribute_uuid']];
if ($s['type'] === '2') {
// remove existing expiration by the same org if it exists
$this->deleteAll(array(
'Sighting.org_id' => $user['org_id'],
'Sighting.type' => 2,
'Sighting.attribute_id' => $attributeId,
));
}
$saveOnBehalfOf = false;
if ($user['Role']['perm_sync']) {
if (isset($s['org_id'])) {
if ($s['org_id'] != 0 && !empty($s['Organisation'])) {
$saveOnBehalfOf = $this->Event->Orgc->captureOrg($s['Organisation'], $user);
} else {
$saveOnBehalfOf = 0;
}
}
}
$this->create();
// TODO: Ignoring possible validation errors
$this->save([
'attribute_id' => $attributeId,
'event_id' => $eventId,
'org_id' => $saveOnBehalfOf === false ? $user['org_id'] : $saveOnBehalfOf,
'date_sighting' => $s['date_sighting'],
'type' => $s['type'],
'source' => $s['source'],
'uuid' => $s['uuid'],
]);
$saved++;
}
if ($saved > 0) {
$this->Event->publishRouter($event['Event']['id'], $passAlong, $user, 'sightings');
}
return $saved;
}
public function pullSightings($user, $server)
{
$HttpSocket = $this->setupHttpSocket($server);
$this->Server = ClassRegistry::init('Server');
try {
$eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, 'sightings');
} catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
return 0;
}
$saved = 0;
// now process the $eventIds to pull each of the events sequentially
// download each event and save sightings
foreach ($eventIds as $k => $eventId) {
try {
$event = $this->Event->downloadEventFromServer($eventId, $server);
} catch (Exception $e) {
$this->logException("Failed downloading the event $eventId from {$server['Server']['name']}.", $e);
continue;
}
$sightings = array();
if (!empty($event) && !empty($event['Event']['Attribute'])) {
foreach ($event['Event']['Attribute'] as $attribute) {
if (!empty($attribute['Sighting'])) {
$sightings = array_merge($sightings, $attribute['Sighting']);
}
}
}
if (!empty($event) && !empty($sightings)) {
$result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $server['Server']['id']);
if (is_numeric($result)) {
$saved += $result;
}
}
}
return $saved;
}
/**
* @return int Timestamp
*/
public function getMaximumRange()
{
$rangeInDays = Configure::read('MISP.Sightings_range');
$rangeInDays = (!empty($rangeInDays) && is_numeric($rangeInDays)) ? $rangeInDays : 365;
return strtotime("-$rangeInDays days");
}
2020-11-11 09:00:50 +01:00
/**
* Sighting reporters setting
* If the event has any sightings for the user's org, then the user is a sighting reporter for the event too.
* This means that he /she has access to the sightings data contained within.
*
* @param int $eventId
* @param int $orgId
* @return bool
*/
private function isReporter($eventId, $orgId)
{
return (bool)$this->find('first', array(
'recursive' => -1,
'callbacks' => false,
'fields' => ['Sighting.id'],
'conditions' => array(
'Sighting.event_id' => $eventId,
'Sighting.org_id' => $orgId,
)
));
}
/**
* Reduce memory usage by not fetching organisation object for every sighting but just once. Then organisation
* object will be deduplicated in memory.
*
* @param int $orgId
* @return array
*/
private function getOrganisationById($orgId)
{
if (isset($this->orgCache[$orgId])) {
return $this->orgCache[$orgId];
}
$org = $this->Organisation->find('first', [
'recursive' => -1,
'conditions' => ['Organisation.id' => $orgId],
'fields' => ['Organisation.id', 'Organisation.uuid', 'Organisation.name']
]);
if (!empty($org)) {
$org = $org['Organisation'];
}
$this->orgCache[$orgId] = $org;
return $this->orgCache[$orgId];
}
/**
* @return int
*/
private function sightingsPolicy()
{
$policy = Configure::read('Plugin.Sightings_policy');
if ($policy === null) { // default policy
return self::SIGHTING_POLICY_EVENT_OWNER;
}
return (int)$policy;
}
2020-11-22 13:39:31 +01:00
private function dateVirtualColumn()
{
if (in_array($this->getDataSource()->config['datasource'], ['Database/Mysql', 'Database/MysqlObserver'], true)) {
return 'DATE(FROM_UNIXTIME(Sighting.date_sighting))';
} else {
return "to_char(date(to_timestamp(Sighting.date_sighting)), 'YYYY-MM-DD')"; // PostgreSQL
}
}
}