MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [sighting] Speedup loading sighting for tags and galaxies

pull/6578/head
Jakub Onderka 2020-11-21 22:13:06 +01:00
parent 8df9bc9116
commit 261d15265d
3 changed files with 94 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/Controller/GalaxyClustersController.php
View File

@ -99,7 +99,6 @@ class GalaxyClustersController extends AppController
$this->paginate['conditions']['AND'][] = $aclConditions;
$this->paginate['contain'] = array_merge($this->paginate['contain'], array('Org', 'Orgc', 'SharingGroup', 'GalaxyClusterRelation', 'TargetingClusterRelation'));
$clusters = $this->paginate();
$sgs = $this->GalaxyCluster->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
foreach ($clusters as $k => $cluster) {
$clusters[$k] = $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters[$k]);
$clusters[$k] = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $clusters[$k]);
@ -122,9 +121,7 @@ class GalaxyClustersController extends AppController
);
}
$tagIds = array();
$sightings = array();
if (!empty($clusters)) {
$galaxyType = $clusters[0]['GalaxyCluster']['type'];
foreach ($clusters as $k => $v) {
$clusters[$k]['event_ids'] = array();
if (!empty($v['Tag'])) {
@ -138,28 +135,10 @@ class GalaxyClustersController extends AppController
}
}
$this->loadModel('Sighting');
$sightings['tags'] = array();
$csvForTags = $this->Sighting->tagsSparkline($tagIds, $this->Auth->user(), '0');
foreach ($clusters as $k => $cluster) {
if (!empty($cluster['GalaxyCluster']['tag_id'])) {
$temp = $this->Sighting->getSightingsForTag($this->Auth->user(), $cluster['GalaxyCluster']['tag_id']);
$clusters[$k]['sightings'] = $temp;
}
}
$csv = array();
foreach ($clusters as $k => $cluster) {
$startDate = !empty($cluster['sightings']) ? min(array_keys($cluster['sightings'])) : date('Y-m-d');
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
$to = date('Y-m-d', time());
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
if (!isset($csv[$k])) {
$csv[$k] = 'Date,Close\n';
}
if (isset($cluster['sightings'][$date])) {
$csv[$k] .= $date . ',' . $cluster['sightings'][$date] . '\n';
} else {
$csv[$k] .= $date . ',0\n';
}
$clusters[$k]['csv'] = $csv[$k];
if (!empty($cluster['GalaxyCluster']['tag_id']) && isset($csvForTags[$cluster['GalaxyCluster']['tag_id']])) {
$clusters[$k]['csv'] = $csvForTags[$cluster['GalaxyCluster']['tag_id']];
}
}
$customClusterCount = $this->GalaxyCluster->fetchGalaxyClusters($this->Auth->user(), [
@ -173,7 +152,6 @@ class GalaxyClustersController extends AppController
$distributionLevels = $this->Attribute->distributionLevels;
unset($distributionLevels[5]);
$this->set('distributionLevels', $distributionLevels);
$this->set('csv', $csv);
$this->set('list', $clusters);
$this->set('galaxy_id', $galaxyId);
$this->set('custom_cluster_count', $customClusterCount);

64
app/Controller/TagsController.php
View File

@ -1,5 +1,4 @@
<?php
App::uses('AppController', 'Controller');
class TagsController extends AppController
@ -12,12 +11,6 @@ class TagsController extends AppController
'Tag.name' => 'asc'
),
'contain' => array(
'EventTag' => array(
'fields' => array('EventTag.event_id')
),
'AttributeTag' => array(
'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
),
'FavouriteTag',
'Organisation' => array(
'fields' => array('id', 'name')
@ -78,37 +71,19 @@ class TagsController extends AppController
}
if ($this->_isRest()) {
unset($this->paginate['limit']);
unset($this->paginate['contain']['EventTag']);
unset($this->paginate['contain']['AttributeTag']);
$paginated = $this->Tag->find('all', $this->paginate);
} else {
if (!empty($passedArgsArray['exclude_statistics'])) {
unset($this->paginate['contain']['EventTag']);
unset($this->paginate['contain']['AttributeTag']);
$this->set('exclude_statistics', true);
}
$paginated = $this->paginate();
}
$tagList = array();
$csv = array();
$sgs = $this->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
foreach ($paginated as $k => $tag) {
$tagList[] = $tag['Tag']['id'];
if (empty($passedArgsArray['exclude_statistics'])) {
$paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
if (!$this->_isRest()) {
$paginated[$k]['event_ids'] = array();
$paginated[$k]['attribute_ids'] = array();
foreach ($paginated[$k]['EventTag'] as $et) {
$paginated[$k]['event_ids'][] = $et['event_id'];
}
unset($paginated[$k]['EventTag']);
foreach ($paginated[$k]['AttributeTag'] as $at) {
$paginated[$k]['attribute_ids'][] = $at['attribute_id'];
}
unset($paginated[$k]['AttributeTag']);
}
$paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
$paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id']);
$paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id']);
}
if (!empty($tag['FavouriteTag'])) {
foreach ($tag['FavouriteTag'] as $ft) {
@ -138,39 +113,11 @@ class TagsController extends AppController
}
if (!$this->_isRest() && empty($passedArgsArray['exclude_statistics'])) {
$this->loadModel('Sighting');
$sightings['event'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList);
$sightings['attribute'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList, 'attribute');
$csvForTags = $this->Sighting->tagsSparkline($tagList, $this->Auth->user(), '0');
foreach ($paginated as $k => $tag) {
$objects = array('event', 'attribute');
foreach ($objects as $object) {
foreach ($tag[$object . '_ids'] as $objectid) {
if (isset($sightings[$object][$objectid])) {
foreach ($sightings[$object][$objectid] as $date => $sightingCount) {
if (!isset($tag['sightings'][$date])) {
$tag['sightings'][$date] = $sightingCount;
} else {
$tag['sightings'][$date] += $sightingCount;
}
}
}
}
if (isset($csvForTags[$tag['Tag']['id']])) {
$paginated[$k]['Tag']['csv'] = $csvForTags[$tag['Tag']['id']];
}
if (!empty($tag['sightings'])) {
$startDate = !empty($tag['sightings']) ? min(array_keys($tag['sightings'])) : date('Y-m-d');
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
$to = date('Y-m-d', time());
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
if (!isset($paginated[$k]['Tag']['csv'])) {
$paginated[$k]['Tag']['csv'] = 'Date,Close\n';
}
if (isset($tag['sightings'][$date])) {
$paginated[$k]['Tag']['csv'] .= $date . ',' . $tag['sightings'][$date] . '\n';
} else {
$paginated[$k]['Tag']['csv'] .= $date . ',0\n';
}
}
}
unset($paginated[$k]['event_ids']);
}
}
if ($this->_isRest()) {
@ -182,7 +129,6 @@ class TagsController extends AppController
} else {
$this->set('passedArgs', json_encode($this->passedArgs));
$this->set('passedArgsArray', $passedArgsArray);
$this->set('csv', $csv);
$this->set('list', $paginated);
$this->set('favouritesOnly', $favouritesOnly);
}

166
app/Model/Sighting.php
View File

@ -5,6 +5,7 @@ App::uses('TmpFileTool', 'Tools');
/**
* @property Attribute $Attribute
* @property Event $Event
* @property Organisation $Organisation
*/
class Sighting extends AppModel
{
@ -203,6 +204,46 @@ class Sighting extends AppModel
return $result;
}
/**
* @param array $tagIds
* @param array $user
* @param null|string $type
* @return array
*/
public function tagsSparkline(array $tagIds, array $user, $type = null)
{
if (empty($tagIds)) {
return [];
}
$conditions = ['Sighting.date_sighting >' => $this->getMaximumRange()];
if ($type !== null) {
$conditions['Sighting.type'] = $type;
}
$sightingsPolicy = $this->sightingsPolicy();
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$conditions['Sighting.org_id'] = $user['org_id'];
}
// TODO: Currently, we dont support `SIGHTING_POLICY_SIGHTING_REPORTER` for tags
$sparklineData = [];
foreach (['event', 'attribute'] as $context) {
$sightings = $this->fetchGroupedSightingsForTags($tagIds, $conditions, $context);
$objectElement = ucfirst($context) . 'Tag';
foreach ($sightings as $sighting) {
$tagId = $sighting[$objectElement]['tag_id'];
$date = $sighting['Sighting']['date_sighting'];
$count = (int)$sighting['Sighting']['sighting_count'];
if (isset($sparklineData[$tagId][$date]['sighting'])) {
$sparklineData[$tagId][$date]['sighting'] += $count;
} else {
$sparklineData[$tagId][$date]['sighting'] = $count;
}
}
}
return $this->generateSparkline($sparklineData, false);
}
/**
* @param array $attributes Attribute must contain Event
* @param array $user
@ -282,7 +323,7 @@ class Sighting extends AppModel
return [];
}
// Returns data in `Y-m-d` format
// Returns date in `Y-m-d` format
$this->virtualFields['date_sighting'] = 'DATE(FROM_UNIXTIME(date_sighting))'; // TODO: Probably just for MySQL
$this->virtualFields['sighting_count'] = 'COUNT(id)';
$groupedSightings = $this->find('all', array(
@ -296,6 +337,39 @@ class Sighting extends AppModel
return $this->attachOrgToSightings($groupedSightings, $user, false);
}
/**
* @param array $tagIds
* @param array $conditions
* @param string $context
* @return array
*/
private function fetchGroupedSightingsForTags(array $tagIds, array $conditions, $context)
{
$conditions[ucfirst($context) . 'Tag.tag_id'] = $tagIds;
// Temporary bind EventTag or AttributeTag model
$this->bindModel([
'hasOne' => [
ucfirst($context) . 'Tag' => [
'foreignKey' => false,
'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id',
]
]
]);
// Returns date in `Y-m-d` format
$this->virtualFields['date_sighting'] = 'DATE(FROM_UNIXTIME(Sighting.date_sighting))'; // TODO: Probably just for MySQL
$this->virtualFields['sighting_count'] = 'COUNT(Sighting.id)';
$sightings = $this->find('all', array(
'recursive' => -1,
'contain' => [ucfirst($context) . 'Tag'],
'conditions' => $conditions,
'fields' => [ucfirst($context) . 'Tag.tag_id', 'date_sighting', 'sighting_count'],
'group' => [ucfirst($context) . 'Tag.id', 'date_sighting'],
'order' => ['date_sighting'], // from oldest
));
unset($this->virtualFields['date_sighting'], $this->virtualFields['sighting_count']);
return $sightings;
}
/**
* @param array $groupedSightings
* @param bool $csvWithFalsePositive
@ -336,6 +410,16 @@ class Sighting extends AppModel
}
}
}
return ['data' => $sightingsData, 'csv' => $this->generateSparkline($sparklineData, $csvWithFalsePositive)];
}
/**
* @param array $sparklineData
* @param bool $csvWithFalsePositive
* @return array
*/
private function generateSparkline(array $sparklineData, $csvWithFalsePositive)
{
$todayString = date('Y-m-d');
$today = strtotime($todayString);
@ -361,8 +445,7 @@ class Sighting extends AppModel
}
$csv[$object] = $csvForObject;
}
return ['data' => $sightingsData, 'csv' => $csv];
return $csv;
}
/**
@ -595,80 +678,6 @@ class Sighting extends AppModel
return $id;
}
public function getSightingsForTag($user, $tag_id, $sgids = array(), $type = false)
{
$conditions = array(
'Sighting.date_sighting >' => $this->getMaximumRange(),
'EventTag.tag_id' => $tag_id
);
if ($type !== false) {
$conditions['Sighting.type'] = $type;
}
$this->bindModel(
array(
'hasOne' => array(
'EventTag' => array(
'className' => 'EventTag',
'foreignKey' => false,
'conditions' => 'EventTag.event_id = Sighting.event_id'
)
)
)
);
$sightings = $this->find('all', array(
'recursive' => -1,
'contain' => array('EventTag'),
'conditions' => $conditions,
'fields' => array('Sighting.id', 'Sighting.event_id', 'Sighting.date_sighting', 'EventTag.tag_id')
));
$sightingsRearranged = array();
foreach ($sightings as $sighting) {
$date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
if (isset($sightingsRearranged[$date])) {
$sightingsRearranged[$date]++;
} else {
$sightingsRearranged[$date] = 1;
}
}
return $sightingsRearranged;
}
/**
* @param $user - not used
* @param array $tagIds
* @param string $context 'event' or 'attribute'
* @param string|false $type
* @return array
*/
public function getSightingsForObjectIds($user, array $tagIds, $context = 'event', $type = '0')
{
$conditions = array(
'Sighting.date_sighting >' => $this->getMaximumRange(),
ucfirst($context) . 'Tag.tag_id' => $tagIds
);
if ($type !== false) {
$conditions['Sighting.type'] = $type;
}
$this->bindModel(array('hasOne' => array(ucfirst($context) . 'Tag' => array('foreignKey' => false, 'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id'))));
$sightings = $this->find('all', array(
'recursive' => -1,
'contain' => array(ucfirst($context) . 'Tag'),
'conditions' => $conditions,
'fields' => array('Sighting.' . $context . '_id', 'Sighting.date_sighting')
));
$sightingsRearranged = array();
foreach ($sightings as $sighting) {
$date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
$contextId = $sighting['Sighting'][$context . '_id'];
if (isset($sightingsRearranged[$contextId][$date])) {
$sightingsRearranged[$contextId][$date]++;
} else {
$sightingsRearranged[$contextId][$date] = 1;
}
}
return $sightingsRearranged;
}
/**
* @param array $user
* @param $ids
@ -1025,9 +1034,6 @@ class Sighting extends AppModel
if (isset($this->orgCache[$orgId])) {
return $this->orgCache[$orgId];
}
if (!isset($this->Organisation)) {
$this->Organisation = ClassRegistry::init('Organisation');
}
$org = $this->Organisation->find('first', [
'recursive' => -1,
'conditions' => ['Organisation.id' => $orgId],