mirror of https://github.com/MISP/MISP
chg: [sighting] Speedup loading sighting for tags and galaxies
parent
8df9bc9116
commit
261d15265d
|
@ -99,7 +99,6 @@ class GalaxyClustersController extends AppController
|
|||
$this->paginate['conditions']['AND'][] = $aclConditions;
|
||||
$this->paginate['contain'] = array_merge($this->paginate['contain'], array('Org', 'Orgc', 'SharingGroup', 'GalaxyClusterRelation', 'TargetingClusterRelation'));
|
||||
$clusters = $this->paginate();
|
||||
$sgs = $this->GalaxyCluster->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
|
||||
foreach ($clusters as $k => $cluster) {
|
||||
$clusters[$k] = $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters[$k]);
|
||||
$clusters[$k] = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $clusters[$k]);
|
||||
|
@ -122,9 +121,7 @@ class GalaxyClustersController extends AppController
|
|||
);
|
||||
}
|
||||
$tagIds = array();
|
||||
$sightings = array();
|
||||
if (!empty($clusters)) {
|
||||
$galaxyType = $clusters[0]['GalaxyCluster']['type'];
|
||||
foreach ($clusters as $k => $v) {
|
||||
$clusters[$k]['event_ids'] = array();
|
||||
if (!empty($v['Tag'])) {
|
||||
|
@ -138,28 +135,10 @@ class GalaxyClustersController extends AppController
|
|||
}
|
||||
}
|
||||
$this->loadModel('Sighting');
|
||||
$sightings['tags'] = array();
|
||||
$csvForTags = $this->Sighting->tagsSparkline($tagIds, $this->Auth->user(), '0');
|
||||
foreach ($clusters as $k => $cluster) {
|
||||
if (!empty($cluster['GalaxyCluster']['tag_id'])) {
|
||||
$temp = $this->Sighting->getSightingsForTag($this->Auth->user(), $cluster['GalaxyCluster']['tag_id']);
|
||||
$clusters[$k]['sightings'] = $temp;
|
||||
}
|
||||
}
|
||||
$csv = array();
|
||||
foreach ($clusters as $k => $cluster) {
|
||||
$startDate = !empty($cluster['sightings']) ? min(array_keys($cluster['sightings'])) : date('Y-m-d');
|
||||
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
|
||||
$to = date('Y-m-d', time());
|
||||
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
|
||||
if (!isset($csv[$k])) {
|
||||
$csv[$k] = 'Date,Close\n';
|
||||
}
|
||||
if (isset($cluster['sightings'][$date])) {
|
||||
$csv[$k] .= $date . ',' . $cluster['sightings'][$date] . '\n';
|
||||
} else {
|
||||
$csv[$k] .= $date . ',0\n';
|
||||
}
|
||||
$clusters[$k]['csv'] = $csv[$k];
|
||||
if (!empty($cluster['GalaxyCluster']['tag_id']) && isset($csvForTags[$cluster['GalaxyCluster']['tag_id']])) {
|
||||
$clusters[$k]['csv'] = $csvForTags[$cluster['GalaxyCluster']['tag_id']];
|
||||
}
|
||||
}
|
||||
$customClusterCount = $this->GalaxyCluster->fetchGalaxyClusters($this->Auth->user(), [
|
||||
|
@ -173,7 +152,6 @@ class GalaxyClustersController extends AppController
|
|||
$distributionLevels = $this->Attribute->distributionLevels;
|
||||
unset($distributionLevels[5]);
|
||||
$this->set('distributionLevels', $distributionLevels);
|
||||
$this->set('csv', $csv);
|
||||
$this->set('list', $clusters);
|
||||
$this->set('galaxy_id', $galaxyId);
|
||||
$this->set('custom_cluster_count', $customClusterCount);
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
<?php
|
||||
|
||||
App::uses('AppController', 'Controller');
|
||||
|
||||
class TagsController extends AppController
|
||||
|
@ -12,12 +11,6 @@ class TagsController extends AppController
|
|||
'Tag.name' => 'asc'
|
||||
),
|
||||
'contain' => array(
|
||||
'EventTag' => array(
|
||||
'fields' => array('EventTag.event_id')
|
||||
),
|
||||
'AttributeTag' => array(
|
||||
'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
|
||||
),
|
||||
'FavouriteTag',
|
||||
'Organisation' => array(
|
||||
'fields' => array('id', 'name')
|
||||
|
@ -78,37 +71,19 @@ class TagsController extends AppController
|
|||
}
|
||||
if ($this->_isRest()) {
|
||||
unset($this->paginate['limit']);
|
||||
unset($this->paginate['contain']['EventTag']);
|
||||
unset($this->paginate['contain']['AttributeTag']);
|
||||
$paginated = $this->Tag->find('all', $this->paginate);
|
||||
} else {
|
||||
if (!empty($passedArgsArray['exclude_statistics'])) {
|
||||
unset($this->paginate['contain']['EventTag']);
|
||||
unset($this->paginate['contain']['AttributeTag']);
|
||||
$this->set('exclude_statistics', true);
|
||||
}
|
||||
$paginated = $this->paginate();
|
||||
}
|
||||
$tagList = array();
|
||||
$csv = array();
|
||||
$sgs = $this->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
|
||||
foreach ($paginated as $k => $tag) {
|
||||
$tagList[] = $tag['Tag']['id'];
|
||||
if (empty($passedArgsArray['exclude_statistics'])) {
|
||||
$paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
|
||||
if (!$this->_isRest()) {
|
||||
$paginated[$k]['event_ids'] = array();
|
||||
$paginated[$k]['attribute_ids'] = array();
|
||||
foreach ($paginated[$k]['EventTag'] as $et) {
|
||||
$paginated[$k]['event_ids'][] = $et['event_id'];
|
||||
}
|
||||
unset($paginated[$k]['EventTag']);
|
||||
foreach ($paginated[$k]['AttributeTag'] as $at) {
|
||||
$paginated[$k]['attribute_ids'][] = $at['attribute_id'];
|
||||
}
|
||||
unset($paginated[$k]['AttributeTag']);
|
||||
}
|
||||
$paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
|
||||
$paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id']);
|
||||
$paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id']);
|
||||
}
|
||||
if (!empty($tag['FavouriteTag'])) {
|
||||
foreach ($tag['FavouriteTag'] as $ft) {
|
||||
|
@ -138,39 +113,11 @@ class TagsController extends AppController
|
|||
}
|
||||
if (!$this->_isRest() && empty($passedArgsArray['exclude_statistics'])) {
|
||||
$this->loadModel('Sighting');
|
||||
$sightings['event'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList);
|
||||
$sightings['attribute'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList, 'attribute');
|
||||
$csvForTags = $this->Sighting->tagsSparkline($tagList, $this->Auth->user(), '0');
|
||||
foreach ($paginated as $k => $tag) {
|
||||
$objects = array('event', 'attribute');
|
||||
foreach ($objects as $object) {
|
||||
foreach ($tag[$object . '_ids'] as $objectid) {
|
||||
if (isset($sightings[$object][$objectid])) {
|
||||
foreach ($sightings[$object][$objectid] as $date => $sightingCount) {
|
||||
if (!isset($tag['sightings'][$date])) {
|
||||
$tag['sightings'][$date] = $sightingCount;
|
||||
} else {
|
||||
$tag['sightings'][$date] += $sightingCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isset($csvForTags[$tag['Tag']['id']])) {
|
||||
$paginated[$k]['Tag']['csv'] = $csvForTags[$tag['Tag']['id']];
|
||||
}
|
||||
if (!empty($tag['sightings'])) {
|
||||
$startDate = !empty($tag['sightings']) ? min(array_keys($tag['sightings'])) : date('Y-m-d');
|
||||
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
|
||||
$to = date('Y-m-d', time());
|
||||
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
|
||||
if (!isset($paginated[$k]['Tag']['csv'])) {
|
||||
$paginated[$k]['Tag']['csv'] = 'Date,Close\n';
|
||||
}
|
||||
if (isset($tag['sightings'][$date])) {
|
||||
$paginated[$k]['Tag']['csv'] .= $date . ',' . $tag['sightings'][$date] . '\n';
|
||||
} else {
|
||||
$paginated[$k]['Tag']['csv'] .= $date . ',0\n';
|
||||
}
|
||||
}
|
||||
}
|
||||
unset($paginated[$k]['event_ids']);
|
||||
}
|
||||
}
|
||||
if ($this->_isRest()) {
|
||||
|
@ -182,7 +129,6 @@ class TagsController extends AppController
|
|||
} else {
|
||||
$this->set('passedArgs', json_encode($this->passedArgs));
|
||||
$this->set('passedArgsArray', $passedArgsArray);
|
||||
$this->set('csv', $csv);
|
||||
$this->set('list', $paginated);
|
||||
$this->set('favouritesOnly', $favouritesOnly);
|
||||
}
|
||||
|
|
|
@ -5,6 +5,7 @@ App::uses('TmpFileTool', 'Tools');
|
|||
/**
|
||||
* @property Attribute $Attribute
|
||||
* @property Event $Event
|
||||
* @property Organisation $Organisation
|
||||
*/
|
||||
class Sighting extends AppModel
|
||||
{
|
||||
|
@ -203,6 +204,46 @@ class Sighting extends AppModel
|
|||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $tagIds
|
||||
* @param array $user
|
||||
* @param null|string $type
|
||||
* @return array
|
||||
*/
|
||||
public function tagsSparkline(array $tagIds, array $user, $type = null)
|
||||
{
|
||||
if (empty($tagIds)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$conditions = ['Sighting.date_sighting >' => $this->getMaximumRange()];
|
||||
if ($type !== null) {
|
||||
$conditions['Sighting.type'] = $type;
|
||||
}
|
||||
$sightingsPolicy = $this->sightingsPolicy();
|
||||
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
|
||||
$conditions['Sighting.org_id'] = $user['org_id'];
|
||||
}
|
||||
// TODO: Currently, we dont support `SIGHTING_POLICY_SIGHTING_REPORTER` for tags
|
||||
$sparklineData = [];
|
||||
foreach (['event', 'attribute'] as $context) {
|
||||
$sightings = $this->fetchGroupedSightingsForTags($tagIds, $conditions, $context);
|
||||
$objectElement = ucfirst($context) . 'Tag';
|
||||
foreach ($sightings as $sighting) {
|
||||
$tagId = $sighting[$objectElement]['tag_id'];
|
||||
$date = $sighting['Sighting']['date_sighting'];
|
||||
$count = (int)$sighting['Sighting']['sighting_count'];
|
||||
|
||||
if (isset($sparklineData[$tagId][$date]['sighting'])) {
|
||||
$sparklineData[$tagId][$date]['sighting'] += $count;
|
||||
} else {
|
||||
$sparklineData[$tagId][$date]['sighting'] = $count;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $this->generateSparkline($sparklineData, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $attributes Attribute must contain Event
|
||||
* @param array $user
|
||||
|
@ -282,7 +323,7 @@ class Sighting extends AppModel
|
|||
return [];
|
||||
}
|
||||
|
||||
// Returns data in `Y-m-d` format
|
||||
// Returns date in `Y-m-d` format
|
||||
$this->virtualFields['date_sighting'] = 'DATE(FROM_UNIXTIME(date_sighting))'; // TODO: Probably just for MySQL
|
||||
$this->virtualFields['sighting_count'] = 'COUNT(id)';
|
||||
$groupedSightings = $this->find('all', array(
|
||||
|
@ -296,6 +337,39 @@ class Sighting extends AppModel
|
|||
return $this->attachOrgToSightings($groupedSightings, $user, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $tagIds
|
||||
* @param array $conditions
|
||||
* @param string $context
|
||||
* @return array
|
||||
*/
|
||||
private function fetchGroupedSightingsForTags(array $tagIds, array $conditions, $context)
|
||||
{
|
||||
$conditions[ucfirst($context) . 'Tag.tag_id'] = $tagIds;
|
||||
// Temporary bind EventTag or AttributeTag model
|
||||
$this->bindModel([
|
||||
'hasOne' => [
|
||||
ucfirst($context) . 'Tag' => [
|
||||
'foreignKey' => false,
|
||||
'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id',
|
||||
]
|
||||
]
|
||||
]);
|
||||
// Returns date in `Y-m-d` format
|
||||
$this->virtualFields['date_sighting'] = 'DATE(FROM_UNIXTIME(Sighting.date_sighting))'; // TODO: Probably just for MySQL
|
||||
$this->virtualFields['sighting_count'] = 'COUNT(Sighting.id)';
|
||||
$sightings = $this->find('all', array(
|
||||
'recursive' => -1,
|
||||
'contain' => [ucfirst($context) . 'Tag'],
|
||||
'conditions' => $conditions,
|
||||
'fields' => [ucfirst($context) . 'Tag.tag_id', 'date_sighting', 'sighting_count'],
|
||||
'group' => [ucfirst($context) . 'Tag.id', 'date_sighting'],
|
||||
'order' => ['date_sighting'], // from oldest
|
||||
));
|
||||
unset($this->virtualFields['date_sighting'], $this->virtualFields['sighting_count']);
|
||||
return $sightings;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $groupedSightings
|
||||
* @param bool $csvWithFalsePositive
|
||||
|
@ -336,6 +410,16 @@ class Sighting extends AppModel
|
|||
}
|
||||
}
|
||||
}
|
||||
return ['data' => $sightingsData, 'csv' => $this->generateSparkline($sparklineData, $csvWithFalsePositive)];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $sparklineData
|
||||
* @param bool $csvWithFalsePositive
|
||||
* @return array
|
||||
*/
|
||||
private function generateSparkline(array $sparklineData, $csvWithFalsePositive)
|
||||
{
|
||||
$todayString = date('Y-m-d');
|
||||
$today = strtotime($todayString);
|
||||
|
||||
|
@ -361,8 +445,7 @@ class Sighting extends AppModel
|
|||
}
|
||||
$csv[$object] = $csvForObject;
|
||||
}
|
||||
|
||||
return ['data' => $sightingsData, 'csv' => $csv];
|
||||
return $csv;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -595,80 +678,6 @@ class Sighting extends AppModel
|
|||
return $id;
|
||||
}
|
||||
|
||||
public function getSightingsForTag($user, $tag_id, $sgids = array(), $type = false)
|
||||
{
|
||||
$conditions = array(
|
||||
'Sighting.date_sighting >' => $this->getMaximumRange(),
|
||||
'EventTag.tag_id' => $tag_id
|
||||
);
|
||||
if ($type !== false) {
|
||||
$conditions['Sighting.type'] = $type;
|
||||
}
|
||||
$this->bindModel(
|
||||
array(
|
||||
'hasOne' => array(
|
||||
'EventTag' => array(
|
||||
'className' => 'EventTag',
|
||||
'foreignKey' => false,
|
||||
'conditions' => 'EventTag.event_id = Sighting.event_id'
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
$sightings = $this->find('all', array(
|
||||
'recursive' => -1,
|
||||
'contain' => array('EventTag'),
|
||||
'conditions' => $conditions,
|
||||
'fields' => array('Sighting.id', 'Sighting.event_id', 'Sighting.date_sighting', 'EventTag.tag_id')
|
||||
));
|
||||
$sightingsRearranged = array();
|
||||
foreach ($sightings as $sighting) {
|
||||
$date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
|
||||
if (isset($sightingsRearranged[$date])) {
|
||||
$sightingsRearranged[$date]++;
|
||||
} else {
|
||||
$sightingsRearranged[$date] = 1;
|
||||
}
|
||||
}
|
||||
return $sightingsRearranged;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $user - not used
|
||||
* @param array $tagIds
|
||||
* @param string $context 'event' or 'attribute'
|
||||
* @param string|false $type
|
||||
* @return array
|
||||
*/
|
||||
public function getSightingsForObjectIds($user, array $tagIds, $context = 'event', $type = '0')
|
||||
{
|
||||
$conditions = array(
|
||||
'Sighting.date_sighting >' => $this->getMaximumRange(),
|
||||
ucfirst($context) . 'Tag.tag_id' => $tagIds
|
||||
);
|
||||
if ($type !== false) {
|
||||
$conditions['Sighting.type'] = $type;
|
||||
}
|
||||
$this->bindModel(array('hasOne' => array(ucfirst($context) . 'Tag' => array('foreignKey' => false, 'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id'))));
|
||||
$sightings = $this->find('all', array(
|
||||
'recursive' => -1,
|
||||
'contain' => array(ucfirst($context) . 'Tag'),
|
||||
'conditions' => $conditions,
|
||||
'fields' => array('Sighting.' . $context . '_id', 'Sighting.date_sighting')
|
||||
));
|
||||
$sightingsRearranged = array();
|
||||
foreach ($sightings as $sighting) {
|
||||
$date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
|
||||
$contextId = $sighting['Sighting'][$context . '_id'];
|
||||
if (isset($sightingsRearranged[$contextId][$date])) {
|
||||
$sightingsRearranged[$contextId][$date]++;
|
||||
} else {
|
||||
$sightingsRearranged[$contextId][$date] = 1;
|
||||
}
|
||||
}
|
||||
return $sightingsRearranged;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $user
|
||||
* @param $ids
|
||||
|
@ -1025,9 +1034,6 @@ class Sighting extends AppModel
|
|||
if (isset($this->orgCache[$orgId])) {
|
||||
return $this->orgCache[$orgId];
|
||||
}
|
||||
if (!isset($this->Organisation)) {
|
||||
$this->Organisation = ClassRegistry::init('Organisation');
|
||||
}
|
||||
$org = $this->Organisation->find('first', [
|
||||
'recursive' => -1,
|
||||
'conditions' => ['Organisation.id' => $orgId],
|
||||
|
|
Loading…
Reference in New Issue