2016-02-29 22:32:04 +01:00
< ? php
App :: uses ( 'AppModel' , 'Model' );
2019-03-09 22:19:37 +01:00
App :: uses ( 'RandomTool' , 'Tools' );
2020-06-20 09:54:32 +02:00
App :: uses ( 'TmpFileTool' , 'Tools' );
2021-10-04 09:45:58 +02:00
App :: uses ( 'FileAccessTool' , 'Tools' );
2016-02-29 22:32:04 +01:00
2018-07-19 11:48:22 +02:00
class Feed extends AppModel
{
2021-01-22 13:01:23 +01:00
public $actsAs = array (
'AuditLog' ,
'SysLogLogable.SysLogLogable' => array (
2018-07-19 11:48:22 +02:00
'change' => 'full'
),
'Trim' ,
'Containable'
);
2016-02-29 22:32:04 +01:00
2018-07-19 11:48:22 +02:00
public $belongsTo = array (
2021-06-24 12:07:09 +02:00
'SharingGroup' => array (
'className' => 'SharingGroup' ,
'foreignKey' => 'sharing_group_id' ,
),
'Tag' => array (
'className' => 'Tag' ,
'foreignKey' => 'tag_id' ,
),
'Orgc' => array (
'className' => 'Organisation' ,
'foreignKey' => 'orgc_id'
)
2018-07-19 11:48:22 +02:00
);
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
public $validate = array (
'url' => array ( // TODO add extra validation to refuse multiple time the same url from the same org
'rule' => array ( 'urlOrExistingFilepath' )
),
'provider' => 'valueNotEmpty' ,
2021-02-09 21:36:36 +01:00
'name' => [
'rule' => 'valueNotEmpty' ,
'required' => true ,
],
2018-07-19 11:48:22 +02:00
'event_id' => array (
'rule' => array ( 'numeric' ),
'message' => 'Please enter a numeric event ID or leave this field blank.' ,
2020-03-30 14:02:14 +02:00
),
'input_source' => array (
'rule' => 'validateInputSource' ,
'message' => ''
2018-07-19 11:48:22 +02:00
)
);
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
// currently we only have an internal name and a display name, but later on we can expand this with versions, default settings, etc
public $feed_types = array (
'misp' => array (
'name' => 'MISP Feed'
),
'freetext' => array (
'name' => 'Freetext Parsed Feed'
),
'csv' => array (
'name' => 'Simple CSV Parsed Feed'
)
);
2016-12-30 16:16:56 +01:00
2021-07-27 16:16:52 +02:00
const DEFAULT_FEED_PULL_RULES = [
2021-06-24 12:07:09 +02:00
'tags' => [
" OR " => [],
" NOT " => [],
],
'orgs' => [
" OR " => [],
" NOT " => [],
],
'url_params' => ''
];
2020-04-17 14:17:54 +02:00
/*
* Cleanup of empty belongsto relationships
*/
public function afterFind ( $results , $primary = false )
{
foreach ( $results as $k => $result ) {
if ( isset ( $result [ 'SharingGroup' ]) && empty ( $result [ 'SharingGroup' ][ 'id' ])) {
unset ( $results [ $k ][ 'SharingGroup' ]);
}
if ( isset ( $result [ 'Tag' ]) && empty ( $result [ 'Tag' ][ 'id' ])) {
unset ( $results [ $k ][ 'Tag' ]);
}
if ( isset ( $result [ 'Orgc' ]) && empty ( $result [ 'Orgc' ][ 'id' ])) {
unset ( $results [ $k ][ 'Orgc' ]);
}
}
return $results ;
}
2021-10-03 17:21:28 +02:00
public function afterSave ( $created , $options = array ())
{
if ( ! $created ) {
2021-10-03 18:54:37 +02:00
$this -> cleanFileCache (( int ) $this -> data [ 'Feed' ][ 'id' ]);
2021-10-03 17:21:28 +02:00
}
}
2020-03-30 14:02:14 +02:00
public function validateInputSource ( $fields )
{
if ( ! empty ( $this -> data [ 'Feed' ][ 'input_source' ])) {
$localAllowed = empty ( Configure :: read ( 'Security.disable_local_feed_access' ));
$validOptions = array ( 'network' );
if ( $localAllowed ) {
$validOptions [] = 'local' ;
}
if ( ! in_array ( $this -> data [ 'Feed' ][ 'input_source' ], $validOptions )) {
return __ (
'Invalid input source. The only valid options are %s. %s' ,
implode ( ', ' , $validOptions ),
( ! $localAllowed && $this -> data [ 'Feed' ][ 'input_source' ] === 'local' ) ?
2021-06-24 12:07:09 +02:00
__ ( 'Security.disable_local_feed_access is currently enabled, local feeds are thereby not allowed.' ) :
''
2020-03-30 14:02:14 +02:00
);
}
}
return true ;
}
2018-07-19 11:48:22 +02:00
public function urlOrExistingFilepath ( $fields )
{
2019-08-26 19:30:53 +02:00
if ( $this -> isFeedLocal ( $this -> data )) {
2018-07-19 11:48:22 +02:00
if ( $this -> data [ 'Feed' ][ 'source_format' ] == 'misp' ) {
if ( ! is_dir ( $this -> data [ 'Feed' ][ 'url' ])) {
return 'For MISP type local feeds, please specify the containing directory.' ;
}
} else {
if ( ! file_exists ( $this -> data [ 'Feed' ][ 'url' ])) {
return 'Invalid path or file not found. Make sure that the path points to an existing file that is readable and watch out for typos.' ;
}
}
} else {
if ( ! filter_var ( $this -> data [ 'Feed' ][ 'url' ], FILTER_VALIDATE_URL )) {
return false ;
}
}
return true ;
}
2016-12-30 16:16:56 +01:00
2018-07-19 11:48:22 +02:00
public function getFeedTypesOptions ()
{
$result = array ();
foreach ( $this -> feed_types as $key => $value ) {
$result [ $key ] = $value [ 'name' ];
}
return $result ;
}
2017-01-24 14:07:55 +01:00
2019-08-25 10:17:50 +02:00
/**
* Gets the event UUIDs from the feed by ID
2020-07-28 17:37:36 +02:00
* Returns an array with the UUIDs of events that are new or that need updating .
*
2019-08-25 10:17:50 +02:00
* @ param array $feed
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket
2019-08-25 10:17:50 +02:00
* @ return array
* @ throws Exception
*/
2020-07-28 17:37:36 +02:00
public function getNewEventUuids ( $feed , HttpSocket $HttpSocket = null )
2018-07-19 11:48:22 +02:00
{
2021-10-03 18:54:37 +02:00
$manifest = $this -> isFeedLocal ( $feed ) ? $this -> downloadManifest ( $feed ) : $this -> getRemoteManifest ( $feed , $HttpSocket );
2018-07-19 11:48:22 +02:00
$this -> Event = ClassRegistry :: init ( 'Event' );
$events = $this -> Event -> find ( 'all' , array (
'conditions' => array (
'Event.uuid' => array_keys ( $manifest ),
),
'recursive' => - 1 ,
2021-10-03 19:56:23 +02:00
'fields' => array ( 'Event.uuid' , 'Event.timestamp' )
2018-07-19 11:48:22 +02:00
));
2019-08-26 09:15:25 +02:00
$result = array ( 'add' => array (), 'edit' => array ());
2018-07-19 11:48:22 +02:00
foreach ( $events as $event ) {
2019-08-26 09:15:25 +02:00
$eventUuid = $event [ 'Event' ][ 'uuid' ];
if ( $event [ 'Event' ][ 'timestamp' ] < $manifest [ $eventUuid ][ 'timestamp' ]) {
2021-10-03 19:56:23 +02:00
$result [ 'edit' ][] = $eventUuid ;
2018-07-19 11:48:22 +02:00
} else {
2019-08-26 09:15:25 +02:00
$this -> __cleanupFile ( $feed , '/' . $eventUuid . '.json' );
2018-07-19 11:48:22 +02:00
}
2019-08-26 09:15:25 +02:00
unset ( $manifest [ $eventUuid ]);
2018-07-19 11:48:22 +02:00
}
2019-08-26 09:15:25 +02:00
// Rest events in manifest does't exists, they will be added
$result [ 'add' ] = array_keys ( $manifest );
2018-07-19 11:48:22 +02:00
return $result ;
}
2016-06-04 01:08:16 +02:00
2019-08-26 09:01:56 +02:00
/**
* @ param array $feed
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2020-09-28 17:00:00 +02:00
* @ return Generator
2019-08-26 09:01:56 +02:00
* @ throws Exception
*/
2020-07-28 17:37:36 +02:00
public function getCache ( array $feed , HttpSocket $HttpSocket = null )
2018-07-19 11:48:22 +02:00
{
2019-08-26 09:01:56 +02:00
$uri = $feed [ 'Feed' ][ 'url' ] . '/hashes.csv' ;
$data = $this -> feedGetUri ( $feed , $uri , $HttpSocket );
if ( empty ( $data )) {
throw new Exception ( " File ' $uri ' with hashes for cache filling is empty. " );
2018-07-19 11:48:22 +02:00
}
2019-08-26 09:01:56 +02:00
2020-06-20 09:54:32 +02:00
// CSV file can be pretty big to do operations in memory, so we save content to temp and iterate line by line.
$tmpFile = new TmpFileTool ();
$tmpFile -> write ( trim ( $data ));
unset ( $data );
2020-10-25 09:44:01 +01:00
return $tmpFile -> intoParsedCsv ();
2018-07-19 11:48:22 +02:00
}
2016-06-04 01:08:16 +02:00
2019-08-25 10:17:50 +02:00
/**
* @ param array $feed
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2019-08-25 10:17:50 +02:00
* @ return array
* @ throws Exception
*/
2020-07-28 17:37:36 +02:00
private function downloadManifest ( $feed , HttpSocket $HttpSocket = null )
2018-07-19 11:48:22 +02:00
{
2019-08-25 10:17:50 +02:00
$manifestUrl = $feed [ 'Feed' ][ 'url' ] . '/manifest.json' ;
2021-10-03 17:03:24 +02:00
$data = $this -> feedGetUri ( $feed , $manifestUrl , $HttpSocket );
2019-08-25 10:17:50 +02:00
2020-07-28 18:33:05 +02:00
try {
return $this -> jsonDecode ( $data );
} catch ( Exception $e ) {
throw new Exception ( " Could not parse ' $manifestUrl ' manifest JSON " , 0 , $e );
2018-07-19 11:48:22 +02:00
}
2019-08-25 10:17:50 +02:00
}
2021-10-03 18:54:37 +02:00
/**
* @ param int $feedId
*/
private function cleanFileCache ( $feedId )
{
$cacheDir = APP . 'tmp' . DS . 'cache' . DS ;
foreach ([ " misp_feed_ { $feedId } _manifest.cache.gz " , " misp_feed_ { $feedId } _manifest.etag " , " misp_feed_ $feedId .cache " , " misp_feed_ $feedId .etag " ] as $fileName ) {
2021-10-04 09:45:58 +02:00
FileAccessTool :: deleteFileIfExists ( $cacheDir . $fileName );
2021-10-03 18:54:37 +02:00
}
}
/**
* Get remote manifest for feed with etag checking .
* @ param array $feed
* @ param HttpSocketExtended $HttpSocket
* @ return array
* @ throws HttpSocketHttpException
* @ throws JsonException
*/
private function getRemoteManifest ( array $feed , HttpSocketExtended $HttpSocket )
{
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . ( int ) $feed [ 'Feed' ][ 'id' ] . '_manifest.cache.gz' ;
$feedCacheEtag = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . ( int ) $feed [ 'Feed' ][ 'id' ] . '_manifest.etag' ;
$etag = null ;
if ( file_exists ( $feedCache ) && file_exists ( $feedCacheEtag )) {
$etag = file_get_contents ( $feedCacheEtag );
}
$manifestUrl = $feed [ 'Feed' ][ 'url' ] . '/manifest.json' ;
try {
$response = $this -> feedGetUriRemote ( $feed , $manifestUrl , $HttpSocket , $etag );
} catch ( HttpSocketHttpException $e ) {
if ( $e -> getCode () === 304 ) { // not modified
$data = file_get_contents ( " compress.zlib:// $feedCache " );
if ( $data === false ) {
return $this -> feedGetUriRemote ( $feed , $manifestUrl , $HttpSocket ) -> json (); // cache file is not readable, fetch without etag
}
return $this -> jsonDecode ( $data );
} else {
throw $e ;
}
}
if ( $response -> getHeader ( 'ETag' )) {
2021-10-04 09:45:58 +02:00
try {
FileAccessTool :: writeCompressedFile ( $feedCache , $response -> body );
FileAccessTool :: writeToFile ( $feedCacheEtag , $response -> getHeader ( 'ETag' ));
} catch ( Exception $e ) {
FileAccessTool :: deleteFileIfExists ( $feedCacheEtag );
$this -> logException ( " Could not save file ` $feedCache ` to cache. " , $e , LOG_NOTICE );
2021-10-03 18:54:37 +02:00
}
} else {
2021-10-04 09:45:58 +02:00
FileAccessTool :: deleteFileIfExists ( $feedCacheEtag );
2021-10-03 18:54:37 +02:00
}
return $response -> json ();
}
2019-08-25 10:17:50 +02:00
/**
* @ param array $feed
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2019-08-25 10:17:50 +02:00
* @ return array
* @ throws Exception
*/
2021-03-10 09:53:23 +01:00
public function getManifest ( array $feed , HttpSocket $HttpSocket = null )
2019-08-25 10:17:50 +02:00
{
2021-10-03 18:54:37 +02:00
$events = $this -> isFeedLocal ( $feed ) ? $this -> downloadManifest ( $feed ) : $this -> getRemoteManifest ( $feed , $HttpSocket );
2018-07-19 11:48:22 +02:00
$events = $this -> __filterEventsIndex ( $events , $feed );
return $events ;
}
2016-06-04 01:08:16 +02:00
2021-10-03 17:03:24 +02:00
/**
* Load remote file with cache support and etag checking .
* @ param array $feed
* @ param HttpSocket $HttpSocket
* @ return string
* @ throws HttpSocketHttpException
*/
private function getFreetextFeedRemote ( array $feed , HttpSocket $HttpSocket )
{
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . ( int ) $feed [ 'Feed' ][ 'id' ] . '.cache' ;
$feedCacheEtag = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . ( int ) $feed [ 'Feed' ][ 'id' ] . '.etag' ;
$etag = null ;
if ( file_exists ( $feedCache )) {
if ( time () - filemtime ( $feedCache ) < 600 ) {
$data = file_get_contents ( $feedCache );
if ( $data !== false ) {
return $data ;
}
} else if ( file_exists ( $feedCacheEtag )) {
$etag = file_get_contents ( $feedCacheEtag );
}
}
try {
$response = $this -> feedGetUriRemote ( $feed , $feed [ 'Feed' ][ 'url' ], $HttpSocket , $etag );
} catch ( HttpSocketHttpException $e ) {
if ( $e -> getCode () === 304 ) { // not modified
$data = file_get_contents ( $feedCache );
if ( $data === false ) {
return $this -> feedGetUriRemote ( $feed , $feed [ 'Feed' ][ 'url' ], $HttpSocket ); // cache file is not readable, fetch without etag
}
return $data ;
} else {
throw $e ;
}
}
2021-10-04 09:45:58 +02:00
try {
FileAccessTool :: writeToFile ( $feedCache , $response -> body );
if ( $response -> getHeader ( 'ETag' )) {
FileAccessTool :: writeToFile ( $feedCacheEtag , $response -> getHeader ( 'ETag' ));
2021-10-03 17:21:28 +02:00
}
2021-10-04 09:45:58 +02:00
} catch ( Exception $e ) {
FileAccessTool :: deleteFileIfExists ( $feedCacheEtag );
$this -> logException ( " Could not save file ` $feedCache ` to cache. " , $e , LOG_NOTICE );
2021-10-03 17:03:24 +02:00
}
return $response -> body ;
}
2019-08-25 11:22:36 +02:00
/**
* @ param array $feed
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2019-08-25 11:22:36 +02:00
* @ param string $type
* @ param int | string $page
* @ param int $limit
* @ param array $params
* @ return array | bool
* @ throws Exception
*/
2020-07-28 17:37:36 +02:00
public function getFreetextFeed ( $feed , HttpSocket $HttpSocket = null , $type = 'freetext' , $page = 1 , $limit = 60 , & $params = array ())
2018-07-19 11:48:22 +02:00
{
2021-10-03 17:03:24 +02:00
if ( $this -> isFeedLocal ( $feed )) {
2019-08-26 19:25:54 +02:00
$feedUrl = $feed [ 'Feed' ][ 'url' ];
2021-10-03 17:03:24 +02:00
$data = $this -> feedGetUri ( $feed , $feedUrl , $HttpSocket );
} else {
$data = $this -> getFreetextFeedRemote ( $feed , $HttpSocket );
2018-07-19 11:48:22 +02:00
}
2019-08-26 19:25:54 +02:00
2018-07-19 11:48:22 +02:00
App :: uses ( 'ComplexTypeTool' , 'Tools' );
$complexTypeTool = new ComplexTypeTool ();
$this -> Warninglist = ClassRegistry :: init ( 'Warninglist' );
$complexTypeTool -> setTLDs ( $this -> Warninglist -> fetchTLDLists ());
$settings = array ();
if ( ! empty ( $feed [ 'Feed' ][ 'settings' ]) && ! is_array ( $feed [ 'Feed' ][ 'settings' ])) {
$feed [ 'Feed' ][ 'settings' ] = json_decode ( $feed [ 'Feed' ][ 'settings' ], true );
}
if ( isset ( $feed [ 'Feed' ][ 'settings' ][ $type ])) {
$settings = $feed [ 'Feed' ][ 'settings' ][ $type ];
}
if ( isset ( $feed [ 'Feed' ][ 'settings' ][ 'common' ])) {
$settings = array_merge ( $settings , $feed [ 'Feed' ][ 'settings' ][ 'common' ]);
}
$resultArray = $complexTypeTool -> checkComplexRouter ( $data , $type , $settings );
$this -> Attribute = ClassRegistry :: init ( 'Attribute' );
foreach ( $resultArray as $key => $value ) {
$resultArray [ $key ][ 'category' ] = $this -> Attribute -> typeDefinitions [ $value [ 'default_type' ]][ 'default_category' ];
}
App :: uses ( 'CustomPaginationTool' , 'Tools' );
$customPagination = new CustomPaginationTool ();
$params = $customPagination -> createPaginationRules ( $resultArray , array ( 'page' => $page , 'limit' => $limit ), 'Feed' , $sort = false );
if ( ! empty ( $page ) && $page != 'all' ) {
$start = ( $page - 1 ) * $limit ;
if ( $start > count ( $resultArray )) {
return false ;
}
$resultArray = array_slice ( $resultArray , $start , $limit );
}
return $resultArray ;
}
2018-01-08 15:56:13 +01:00
2018-07-19 11:48:22 +02:00
public function getFreetextFeedCorrelations ( $data , $feedId )
{
$values = array ();
foreach ( $data as $key => $value ) {
$values [] = $value [ 'value' ];
}
$this -> Attribute = ClassRegistry :: init ( 'Attribute' );
$redis = $this -> setupRedis ();
if ( $redis !== false ) {
$feeds = $this -> find ( 'all' , array (
'recursive' => - 1 ,
'conditions' => array ( 'Feed.id !=' => $feedId ),
'fields' => array ( 'id' , 'name' , 'url' , 'provider' , 'source_format' )
));
foreach ( $feeds as $k => $v ) {
if ( ! $redis -> exists ( 'misp:feed_cache:' . $v [ 'Feed' ][ 'id' ])) {
unset ( $feeds [ $k ]);
}
}
} else {
return array ();
}
// Adding a 3rd parameter to a list find seems to allow grouping several results into a key. If we ran a normal list with value => event_id we'd only get exactly one entry for each value
// The cost of this method is orders of magnitude lower than getting all id - event_id - value triplets and then doing a double loop comparison
$correlations = $this -> Attribute -> find ( 'list' , array ( 'conditions' => array ( 'Attribute.value1' => $values , 'Attribute.deleted' => 0 ), 'fields' => array ( 'Attribute.event_id' , 'Attribute.event_id' , 'Attribute.value1' )));
$correlations2 = $this -> Attribute -> find ( 'list' , array ( 'conditions' => array ( 'Attribute.value2' => $values , 'Attribute.deleted' => 0 ), 'fields' => array ( 'Attribute.event_id' , 'Attribute.event_id' , 'Attribute.value2' )));
$correlations = array_merge_recursive ( $correlations , $correlations2 );
foreach ( $data as $key => $value ) {
if ( isset ( $correlations [ $value [ 'value' ]])) {
$data [ $key ][ 'correlations' ] = array_values ( $correlations [ $value [ 'value' ]]);
}
if ( $redis ) {
foreach ( $feeds as $k => $v ) {
if ( $redis -> sismember ( 'misp:feed_cache:' . $v [ 'Feed' ][ 'id' ], md5 ( $value [ 'value' ]))) {
$data [ $key ][ 'feed_correlations' ][] = array ( $v );
}
}
}
}
return $data ;
}
2016-12-30 16:16:56 +01:00
2020-05-13 13:03:39 +02:00
/**
* Attach correlations from cached servers or feeds .
*
2020-09-28 12:24:35 +02:00
* @ param array $attributes
2020-05-13 13:03:39 +02:00
* @ param array $user
* @ param array $event
2020-09-28 19:52:19 +02:00
* @ param bool $overrideLimit Override hardcoded limit for 10 000 correlations .
2020-05-13 13:03:39 +02:00
* @ param string $scope `Feed` or `Server`
* @ return array
*/
2020-09-28 12:24:35 +02:00
public function attachFeedCorrelations ( array $attributes , array $user , array & $event , $overrideLimit = false , $scope = 'Feed' )
2018-07-19 11:48:22 +02:00
{
2020-09-28 12:24:35 +02:00
if ( empty ( $attributes )) {
return $attributes ;
}
2020-05-13 13:03:39 +02:00
try {
$redis = $this -> setupRedisWithException ();
} catch ( Exception $e ) {
2020-09-28 12:24:35 +02:00
return $attributes ;
2020-05-13 13:03:39 +02:00
}
2019-08-18 09:21:11 +02:00
2020-05-13 13:03:39 +02:00
$cachePrefix = 'misp:' . strtolower ( $scope ) . '_cache:' ;
2019-08-16 22:02:33 +02:00
2020-09-28 12:24:35 +02:00
// Skip if redis cache for $scope is empty.
2020-05-13 13:03:39 +02:00
if ( $redis -> sCard ( $cachePrefix . 'combined' ) === 0 ) {
2020-09-28 12:24:35 +02:00
return $attributes ;
}
if ( ! isset ( $this -> Attribute )) {
$this -> Attribute = ClassRegistry :: init ( 'Attribute' );
2020-05-13 13:03:39 +02:00
}
2020-09-28 12:24:35 +02:00
$compositeTypes = $this -> Attribute -> getCompositeTypes ();
2020-05-13 13:03:39 +02:00
$pipe = $redis -> multi ( Redis :: PIPELINE );
2020-09-28 12:24:35 +02:00
$hashTable = [];
$redisResultToAttributePosition = [];
2020-05-13 13:03:39 +02:00
2020-09-28 12:24:35 +02:00
foreach ( $attributes as $k => $attribute ) {
2021-07-27 15:19:41 +02:00
if ( in_array ( $attribute [ 'type' ], Attribute :: NON_CORRELATING_TYPES , true )) {
2020-09-28 12:24:35 +02:00
continue ; // attribute type is not correlateable
}
if ( ! empty ( $attribute [ 'disable_correlation' ])) {
continue ; // attribute correlation is disabled
}
2020-05-13 13:03:39 +02:00
2021-01-10 17:44:37 +01:00
if ( in_array ( $attribute [ 'type' ], $compositeTypes , true )) {
2020-09-28 12:24:35 +02:00
list ( $value1 , $value2 ) = explode ( '|' , $attribute [ 'value' ]);
$parts = [ $value1 ];
2021-07-21 08:42:05 +02:00
if ( ! in_array ( $attribute [ 'type' ], Attribute :: PRIMARY_ONLY_CORRELATING_TYPES , true )) {
2020-09-28 12:24:35 +02:00
$parts [] = $value2 ;
}
2020-05-13 13:03:39 +02:00
} else {
2020-09-28 12:24:35 +02:00
$parts = [ $attribute [ 'value' ]];
2020-10-29 18:55:17 +01:00
// Some feeds contains URL without protocol, so if attribute is URL and value contains protocol,
// we will check also value without protocol.
if ( $attribute [ 'type' ] === 'url' || $attribute [ 'type' ] === 'uri' ) {
$protocolPos = strpos ( $attribute [ 'value' ], '://' );
if ( $protocolPos !== false ) {
$parts [] = substr ( $attribute [ 'value' ], $protocolPos + 3 );
}
}
2020-09-28 12:24:35 +02:00
}
foreach ( $parts as $part ) {
$md5 = md5 ( $part );
$hashTable [] = $md5 ;
$redis -> sismember ( $cachePrefix . 'combined' , $md5 );
$redisResultToAttributePosition [] = $k ;
2018-07-19 11:48:22 +02:00
}
2020-05-13 13:03:39 +02:00
}
2020-09-28 12:24:35 +02:00
if ( empty ( $redisResultToAttributePosition )) {
// No attribute that can be correlated
$pipe -> discard ();
return $attributes ;
}
2020-05-13 13:03:39 +02:00
$results = $pipe -> exec ();
2020-09-28 12:24:35 +02:00
$hitIds = [];
foreach ( $results as $k => $result ) {
if ( $result ) {
$hitIds [] = $k ;
}
}
if ( empty ( $hitIds )) {
return $attributes ; // nothing matches, skip
}
2020-09-28 19:52:19 +02:00
$hitCount = count ( $hitIds );
if ( ! $overrideLimit && $hitCount > 10000 ) {
$event [ 'FeedCount' ] = $hitCount ;
foreach ( $hitIds as $k ) {
$attributes [ $redisResultToAttributePosition [ $k ]][ 'FeedHit' ] = true ;
}
return $attributes ;
}
2020-09-28 15:41:32 +02:00
$sources = $this -> getCachedFeedsOrServers ( $user , $scope );
2020-09-28 12:24:35 +02:00
foreach ( $sources as $source ) {
$sourceId = $source [ $scope ][ 'id' ];
$pipe = $redis -> multi ( Redis :: PIPELINE );
foreach ( $hitIds as $k ) {
$redis -> sismember ( $cachePrefix . $sourceId , $hashTable [ $k ]);
}
$sourceHits = $pipe -> exec ();
$sourceHasHit = false ;
foreach ( $sourceHits as $k => $hit ) {
if ( $hit ) {
if ( ! isset ( $event [ $scope ][ $sourceId ])) {
$event [ $scope ][ $sourceId ] = $source [ $scope ];
}
2021-01-02 18:30:02 +01:00
2020-09-28 12:24:35 +02:00
$attributePosition = $redisResultToAttributePosition [ $hitIds [ $k ]];
2021-01-02 18:30:02 +01:00
$alreadyAttached = isset ( $attributes [ $attributePosition ][ $scope ]) &&
in_array ( $sourceId , array_column ( $attributes [ $attributePosition ][ $scope ], 'id' ));
if ( ! $alreadyAttached ) {
$attributes [ $attributePosition ][ $scope ][] = $source [ $scope ];
}
2020-09-28 12:24:35 +02:00
$sourceHasHit = true ;
2018-07-19 11:48:22 +02:00
}
2020-05-13 13:03:39 +02:00
}
2021-01-02 18:30:02 +01:00
// Append also exact MISP feed or server event UUID
2020-09-28 12:24:35 +02:00
// TODO: This can be optimised in future to do that in one pass
2020-10-16 14:41:45 +02:00
if ( $sourceHasHit && ( $scope === 'Server' || $source [ $scope ][ 'source_format' ] === 'misp' )) {
2020-05-13 13:03:39 +02:00
$pipe = $redis -> multi ( Redis :: PIPELINE );
2020-09-28 12:24:35 +02:00
$eventUuidHitPosition = [];
foreach ( $hitIds as $sourceHitPos => $k ) {
if ( $sourceHits [ $sourceHitPos ]) {
$redis -> smembers ( $cachePrefix . 'event_uuid_lookup:' . $hashTable [ $k ]);
$eventUuidHitPosition [] = $redisResultToAttributePosition [ $k ];
2018-07-19 11:48:22 +02:00
}
}
2020-09-28 12:24:35 +02:00
$mispFeedHits = $pipe -> exec ();
foreach ( $mispFeedHits as $sourceHitPos => $feedUuidMatches ) {
if ( empty ( $feedUuidMatches )) {
continue ;
2018-07-19 11:48:22 +02:00
}
2020-09-28 12:24:35 +02:00
foreach ( $feedUuidMatches as $url ) {
list ( $feedId , $eventUuid ) = explode ( '/' , $url );
if ( $feedId != $sourceId ) {
continue ; // just process current source, skip others
}
if ( empty ( $event [ $scope ][ $feedId ][ 'event_uuids' ]) || ! in_array ( $eventUuid , $event [ $scope ][ $feedId ][ 'event_uuids' ])) {
$event [ $scope ][ $feedId ][ 'event_uuids' ][] = $eventUuid ;
}
$attributePosition = $eventUuidHitPosition [ $sourceHitPos ];
foreach ( $attributes [ $attributePosition ][ $scope ] as $tempKey => $tempFeed ) {
if ( $tempFeed [ 'id' ] == $feedId ) {
2021-01-02 18:30:02 +01:00
if ( empty ( $attributes [ $attributePosition ][ $scope ][ $tempKey ][ 'event_uuids' ]) || ! in_array ( $eventUuid , $attributes [ $attributePosition ][ $scope ][ $tempKey ][ 'event_uuids' ])) {
$attributes [ $attributePosition ][ $scope ][ $tempKey ][ 'event_uuids' ][] = $eventUuid ;
}
2020-09-28 12:24:35 +02:00
break ;
2018-07-19 11:48:22 +02:00
}
}
}
}
}
}
2020-05-13 13:03:39 +02:00
2020-09-28 17:00:00 +02:00
if ( isset ( $event [ $scope ])) {
2019-01-18 15:33:04 +01:00
$event [ $scope ] = array_values ( $event [ $scope ]);
2018-07-19 11:48:22 +02:00
}
2020-05-13 13:03:39 +02:00
2020-09-28 12:24:35 +02:00
return $attributes ;
2018-07-19 11:48:22 +02:00
}
2016-06-04 01:08:16 +02:00
2020-09-28 15:41:32 +02:00
/**
* Return just feeds or servers that has some data in Redis cache .
* @ param array $user
* @ param string $scope 'Feed' or 'Server'
* @ return array
*/
private function getCachedFeedsOrServers ( array $user , $scope )
{
if ( $scope === 'Feed' ) {
$params = array (
'recursive' => - 1 ,
'fields' => array ( 'id' , 'name' , 'url' , 'provider' , 'source_format' )
);
if ( ! $user [ 'Role' ][ 'perm_site_admin' ]) {
$params [ 'conditions' ] = array ( 'Feed.lookup_visible' => 1 );
}
$sources = $this -> find ( 'all' , $params );
} else {
$params = array (
'recursive' => - 1 ,
'fields' => array ( 'id' , 'name' , 'url' )
);
if ( ! $user [ 'Role' ][ 'perm_site_admin' ]) {
$params [ 'conditions' ] = array ( 'Server.caching_enabled' => 1 );
}
$this -> Server = ClassRegistry :: init ( 'Server' );
$sources = $this -> Server -> find ( 'all' , $params );
}
try {
$redis = $this -> setupRedisWithException ();
$pipe = $redis -> multi ( Redis :: PIPELINE );
$cachePrefix = 'misp:' . strtolower ( $scope ) . '_cache:' ;
foreach ( $sources as $source ) {
$pipe -> exists ( $cachePrefix . $source [ $scope ][ 'id' ]);
}
$results = $pipe -> exec ();
foreach ( $sources as $k => $source ) {
if ( ! $results [ $k ]) {
unset ( $sources [ $k ]);
}
}
2021-06-24 12:07:09 +02:00
} catch ( Exception $e ) {
}
2020-09-28 15:41:32 +02:00
return $sources ;
}
2021-05-05 17:20:40 +02:00
/**
* @ param array $actions
* @ param array $feed
* @ param HttpSocket | null $HttpSocket
* @ param array $user
* @ param int | false $jobId
* @ return array
* @ throws Exception
*/
private function downloadFromFeed ( array $actions , array $feed , HttpSocket $HttpSocket = null , array $user , $jobId = false )
2018-07-19 11:48:22 +02:00
{
2019-08-26 09:15:25 +02:00
$total = count ( $actions [ 'add' ]) + count ( $actions [ 'edit' ]);
2018-07-19 11:48:22 +02:00
$currentItem = 0 ;
$this -> Event = ClassRegistry :: init ( 'Event' );
$results = array ();
2019-08-25 12:13:13 +02:00
$filterRules = $this -> __prepareFilterRules ( $feed );
2019-08-26 09:15:25 +02:00
foreach ( $actions [ 'add' ] as $uuid ) {
try {
$result = $this -> __addEventFromFeed ( $HttpSocket , $feed , $uuid , $user , $filterRules );
2021-05-05 17:20:40 +02:00
if ( $result === true ) {
2019-08-27 17:59:33 +02:00
$results [ 'add' ][ 'success' ] = $uuid ;
2021-05-05 17:20:40 +02:00
} else if ( $result !== 'blocked' ) {
$results [ 'add' ][ 'fail' ] = [ 'uuid' => $uuid , 'reason' => $result ];
$this -> log ( " Could not add event ' $uuid ' from feed { $feed [ 'Feed' ][ 'id' ] } : $result " , LOG_WARNING );
2019-08-27 17:59:33 +02:00
}
2019-08-26 09:15:25 +02:00
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not add event ' $uuid ' from feed { $feed [ 'Feed' ][ 'id' ] } . " , $e );
2019-08-26 09:15:25 +02:00
$results [ 'add' ][ 'fail' ] = array ( 'uuid' => $uuid , 'reason' => $e -> getMessage ());
2018-07-19 11:48:22 +02:00
}
2019-08-27 17:59:33 +02:00
2019-08-26 09:15:25 +02:00
$this -> __cleanupFile ( $feed , '/' . $uuid . '.json' );
2019-08-26 19:47:25 +02:00
$this -> jobProgress ( $jobId , null , 100 * (( $currentItem + 1 ) / $total ));
2019-08-26 09:15:25 +02:00
$currentItem ++ ;
2018-07-19 11:48:22 +02:00
}
2019-08-25 12:23:33 +02:00
2021-10-03 19:56:23 +02:00
foreach ( $actions [ 'edit' ] as $uuid ) {
2019-08-26 09:15:25 +02:00
try {
2021-10-03 19:56:23 +02:00
$result = $this -> __updateEventFromFeed ( $HttpSocket , $feed , $uuid , $user , $filterRules );
2021-05-05 17:20:40 +02:00
if ( $result === true ) {
$results [ 'add' ][ 'success' ] = $uuid ;
} else if ( $result !== 'blocked' ) {
$results [ 'add' ][ 'fail' ] = [ 'uuid' => $uuid , 'reason' => $result ];
2021-08-17 14:05:41 +02:00
$this -> log ( " Could not edit event ' $uuid ' from feed { $feed [ 'Feed' ][ 'id' ] } : " . json_encode ( $result ), LOG_WARNING );
2019-08-27 17:59:33 +02:00
}
2019-08-26 09:15:25 +02:00
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not edit event ' $uuid ' from feed { $feed [ 'Feed' ][ 'id' ] } . " , $e );
2019-08-26 09:15:25 +02:00
$results [ 'edit' ][ 'fail' ] = array ( 'uuid' => $uuid , 'reason' => $e -> getMessage ());
}
$this -> __cleanupFile ( $feed , '/' . $uuid . '.json' );
2021-05-05 17:20:40 +02:00
if ( $currentItem % 10 === 0 ) {
2019-08-26 19:47:25 +02:00
$this -> jobProgress ( $jobId , null , 100 * (( $currentItem + 1 ) / $total ));
2019-08-26 09:15:25 +02:00
}
$currentItem ++ ;
2018-07-19 11:48:22 +02:00
}
2019-08-26 09:15:25 +02:00
2018-07-19 11:48:22 +02:00
return $results ;
}
2017-05-08 14:22:27 +02:00
2018-07-19 11:48:22 +02:00
private function __createFeedRequest ( $headers = false )
{
$version = $this -> checkMISPVersion ();
$version = implode ( '.' , $version );
2019-08-24 10:52:47 +02:00
2018-07-19 11:48:22 +02:00
$result = array (
'header' => array (
2020-07-21 10:10:08 +02:00
'Accept' => array ( 'application/json' , 'text/plain' , 'text/*' ),
2020-04-28 15:31:27 +02:00
'MISP-version' => $version ,
'MISP-uuid' => Configure :: read ( 'MISP.uuid' ),
2018-07-19 11:48:22 +02:00
)
);
2020-04-28 15:31:27 +02:00
2020-09-26 19:53:52 +02:00
$commit = $this -> checkMIPSCommit ();
2018-07-19 11:48:22 +02:00
if ( $commit ) {
$result [ 'header' ][ 'commit' ] = $commit ;
}
if ( ! empty ( $headers )) {
$lines = explode ( " \n " , $headers );
foreach ( $lines as $line ) {
if ( ! empty ( $line )) {
$kv = explode ( ':' , $line );
if ( ! empty ( $kv [ 0 ]) && ! empty ( $kv [ 1 ])) {
if ( ! in_array ( $kv [ 0 ], array ( 'commit' , 'MISP-version' , 'MISP-uuid' ))) {
$result [ 'header' ][ trim ( $kv [ 0 ])] = trim ( $kv [ 1 ]);
}
}
}
}
}
return $result ;
}
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
private function __checkIfEventBlockedByFilter ( $event , $filterRules )
{
$fields = array ( 'tags' => 'Tag' , 'orgs' => 'Orgc' );
$prefixes = array ( 'OR' , 'NOT' );
foreach ( $fields as $field => $fieldModel ) {
foreach ( $prefixes as $prefix ) {
if ( ! empty ( $filterRules [ $field ][ $prefix ])) {
$found = false ;
if ( isset ( $event [ 'Event' ][ $fieldModel ]) && ! empty ( $event [ 'Event' ][ $fieldModel ])) {
if ( ! isset ( $event [ 'Event' ][ $fieldModel ][ 0 ])) {
$event [ 'Event' ][ $fieldModel ] = array ( 0 => $event [ 'Event' ][ $fieldModel ]);
}
foreach ( $event [ 'Event' ][ $fieldModel ] as $object ) {
foreach ( $filterRules [ $field ][ $prefix ] as $temp ) {
if ( stripos ( $object [ 'name' ], $temp ) !== false ) {
$found = true ;
2019-09-03 18:04:47 +02:00
break 2 ;
2018-07-19 11:48:22 +02:00
}
}
}
}
if ( $prefix === 'OR' && ! $found ) {
return false ;
}
if ( $prefix !== 'OR' && $found ) {
return false ;
}
}
}
}
return true ;
}
2016-06-04 01:08:16 +02:00
2021-03-10 09:53:23 +01:00
private function __filterEventsIndex ( array $events , array $feed )
2018-07-19 11:48:22 +02:00
{
2019-08-25 12:13:13 +02:00
$filterRules = $this -> __prepareFilterRules ( $feed );
if ( ! $filterRules ) {
$filterRules = array ();
2018-07-19 11:48:22 +02:00
}
foreach ( $events as $k => $event ) {
2021-03-10 09:53:23 +01:00
if ( isset ( $event [ 'orgc' ]) && ! isset ( $event [ 'Orgc' ])) { // fix key case
$event [ 'Orgc' ] = $event [ 'orgc' ];
unset ( $event [ 'orgc' ]);
$events [ $k ] = $event ;
}
2018-07-19 11:48:22 +02:00
if ( isset ( $filterRules [ 'orgs' ][ 'OR' ]) && ! empty ( $filterRules [ 'orgs' ][ 'OR' ]) && ! in_array ( $event [ 'Orgc' ][ 'name' ], $filterRules [ 'orgs' ][ 'OR' ])) {
unset ( $events [ $k ]);
continue ;
}
if ( isset ( $filterRules [ 'orgs' ][ 'NO' ]) && ! empty ( $filterRules [ 'orgs' ][ 'NOT' ]) && in_array ( $event [ 'Orgc' ][ 'name' ], $filterRules [ 'orgs' ][ 'OR' ])) {
unset ( $events [ $k ]);
continue ;
}
if ( isset ( $filterRules [ 'tags' ][ 'OR' ]) && ! empty ( $filterRules [ 'tags' ][ 'OR' ])) {
if ( ! isset ( $event [ 'Tag' ]) || empty ( $event [ 'Tag' ])) {
unset ( $events [ $k ]);
}
$found = false ;
foreach ( $event [ 'Tag' ] as $tag ) {
foreach ( $filterRules [ 'tags' ][ 'OR' ] as $filterTag ) {
if ( strpos ( strtolower ( $tag [ 'name' ]), strtolower ( $filterTag ))) {
$found = true ;
}
}
}
if ( ! $found ) {
unset ( $k );
continue ;
}
}
if ( isset ( $filterRules [ 'tags' ][ 'NOT' ]) && ! empty ( $filterRules [ 'tags' ][ 'NOT' ])) {
if ( isset ( $event [ 'Tag' ]) && ! empty ( $event [ 'Tag' ])) {
$found = false ;
foreach ( $event [ 'Tag' ] as $tag ) {
foreach ( $filterRules [ 'tags' ][ 'NOT' ] as $filterTag ) {
if ( strpos ( strtolower ( $tag [ 'name' ]), strtolower ( $filterTag ))) {
$found = true ;
}
}
}
if ( $found ) {
unset ( $k );
}
}
}
}
return $events ;
}
2016-06-04 01:08:16 +02:00
2019-08-25 12:23:33 +02:00
/**
* @ param array $feed
* @ param string $uuid
2020-06-26 14:56:38 +02:00
* @ param array $user
2019-08-25 12:23:33 +02:00
* @ return array | bool
* @ throws Exception
*/
2020-06-26 14:56:38 +02:00
public function downloadAndSaveEventFromFeed ( array $feed , $uuid , array $user )
2018-07-19 11:48:22 +02:00
{
2020-06-26 14:56:38 +02:00
$event = $this -> downloadEventFromFeed ( $feed , $uuid );
2018-07-19 11:48:22 +02:00
if ( ! is_array ( $event ) || isset ( $event [ 'code' ])) {
return false ;
}
return $this -> __saveEvent ( $event , $user );
}
2016-06-04 01:08:16 +02:00
2019-08-25 12:23:33 +02:00
/**
* @ param array $feed
* @ param string $uuid
* @ return bool | string | array
* @ throws Exception
*/
2020-06-26 14:56:38 +02:00
public function downloadEventFromFeed ( array $feed , $uuid )
2018-07-19 11:48:22 +02:00
{
2019-08-28 17:09:28 +02:00
$filerRules = $this -> __prepareFilterRules ( $feed );
2020-07-28 17:37:36 +02:00
$HttpSocket = $this -> isFeedLocal ( $feed ) ? null : $this -> __setupHttpSocket ( $feed );
2019-08-25 12:23:33 +02:00
$event = $this -> downloadAndParseEventFromFeed ( $feed , $uuid , $HttpSocket );
2019-08-28 17:09:28 +02:00
return $this -> __prepareEvent ( $event , $feed , $filerRules );
2018-07-19 11:48:22 +02:00
}
2016-06-04 01:08:16 +02:00
2020-06-26 14:56:38 +02:00
/**
* @ param array $event
* @ param array $user
* @ return array
*/
private function __saveEvent ( array $event , array $user )
2018-07-19 11:48:22 +02:00
{
$this -> Event = ClassRegistry :: init ( 'Event' );
$existingEvent = $this -> Event -> find ( 'first' , array (
2021-06-24 12:07:09 +02:00
'conditions' => array ( 'Event.uuid' => $event [ 'Event' ][ 'uuid' ]),
'recursive' => - 1 ,
'fields' => array ( 'Event.uuid' , 'Event.id' , 'Event.timestamp' )
2018-07-19 11:48:22 +02:00
));
$result = array ();
if ( ! empty ( $existingEvent )) {
$result [ 'action' ] = 'edit' ;
if ( $existingEvent [ 'Event' ][ 'timestamp' ] < $event [ 'Event' ][ 'timestamp' ]) {
2020-06-26 14:56:38 +02:00
$result [ 'result' ] = $this -> Event -> _edit ( $event , $user );
2018-07-19 11:48:22 +02:00
} else {
$result [ 'result' ] = 'No change' ;
}
} else {
$result [ 'action' ] = 'add' ;
$result [ 'result' ] = $this -> Event -> _add ( $event , true , $user );
}
return $result ;
}
2016-06-04 01:08:16 +02:00
2021-03-10 09:53:23 +01:00
/**
* @ param array $event
* @ param array $feed
* @ param array $filterRules
* @ return array | string
*/
private function __prepareEvent ( $event , array $feed , $filterRules )
2018-07-19 11:48:22 +02:00
{
if ( isset ( $event [ 'response' ])) {
$event = $event [ 'response' ];
}
if ( isset ( $event [ 0 ])) {
$event = $event [ 0 ];
}
if ( ! isset ( $event [ 'Event' ][ 'uuid' ])) {
2021-03-10 09:53:23 +01:00
throw new InvalidArgumentException ( " Event UUID field missing. " );
}
if ( isset ( $event [ 'Event' ][ 'orgc' ]) && ! isset ( $event [ 'Event' ][ 'Orgc' ])) { // fix key case
$event [ 'Event' ][ 'Orgc' ] = $event [ 'Event' ][ 'orgc' ];
unset ( $event [ 'Event' ][ 'orgc' ]);
2018-07-19 11:48:22 +02:00
}
$event [ 'Event' ][ 'distribution' ] = $feed [ 'Feed' ][ 'distribution' ];
$event [ 'Event' ][ 'sharing_group_id' ] = $feed [ 'Feed' ][ 'sharing_group_id' ];
if ( ! empty ( $event [ 'Event' ][ 'Attribute' ])) {
foreach ( $event [ 'Event' ][ 'Attribute' ] as $key => $attribute ) {
$event [ 'Event' ][ 'Attribute' ][ $key ][ 'distribution' ] = 5 ;
}
}
if ( $feed [ 'Feed' ][ 'tag_id' ]) {
if ( ! isset ( $event [ 'Event' ][ 'Tag' ])) {
$event [ 'Event' ][ 'Tag' ] = array ();
}
$found = false ;
2019-08-25 10:17:50 +02:00
foreach ( $event [ 'Event' ][ 'Tag' ] as $tag ) {
if ( strtolower ( $tag [ 'name' ]) === strtolower ( $feed [ 'Tag' ][ 'name' ])) {
$found = true ;
break ;
2018-07-19 11:48:22 +02:00
}
}
if ( ! $found ) {
$feedTag = $this -> Tag -> find ( 'first' , array ( 'conditions' => array ( 'Tag.id' => $feed [ 'Feed' ][ 'tag_id' ]), 'recursive' => - 1 , 'fields' => array ( 'Tag.name' , 'Tag.colour' , 'Tag.exportable' )));
if ( ! empty ( $feedTag )) {
$event [ 'Event' ][ 'Tag' ][] = $feedTag [ 'Tag' ];
}
}
}
if ( $feed [ 'Feed' ][ 'sharing_group_id' ]) {
$sg = $this -> SharingGroup -> find ( 'first' , array (
2021-06-24 12:07:09 +02:00
'recursive' => - 1 ,
'conditions' => array ( 'SharingGroup.id' => $feed [ 'Feed' ][ 'sharing_group_id' ])
2018-07-19 11:48:22 +02:00
));
if ( ! empty ( $sg )) {
$event [ 'Event' ][ 'SharingGroup' ] = $sg [ 'SharingGroup' ];
} else {
// We have an SG ID for the feed, but the SG is gone. Make the event private as a fall-back.
$event [ 'Event' ][ 'distribution' ] = 0 ;
$event [ 'Event' ][ 'sharing_group_id' ] = 0 ;
}
}
if ( ! $this -> __checkIfEventBlockedByFilter ( $event , $filterRules )) {
return 'blocked' ;
}
return $event ;
}
2016-06-04 01:08:16 +02:00
2019-08-28 17:09:28 +02:00
/**
* @ param array $feed
* @ return bool | mixed
* @ throws Exception
*/
2018-07-19 11:48:22 +02:00
private function __prepareFilterRules ( $feed )
{
$filterRules = false ;
if ( isset ( $feed [ 'Feed' ][ 'rules' ]) && ! empty ( $feed [ 'Feed' ][ 'rules' ])) {
$filterRules = json_decode ( $feed [ 'Feed' ][ 'rules' ], true );
2019-08-28 17:09:28 +02:00
if ( $filterRules === null ) {
throw new Exception ( 'Could not parse feed filter rules JSON: ' . json_last_error_msg (), json_last_error ());
}
2018-07-19 11:48:22 +02:00
}
return $filterRules ;
}
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
private function __setupHttpSocket ( $feed )
{
App :: uses ( 'SyncTool' , 'Tools' );
$syncTool = new SyncTool ();
2021-10-03 17:03:24 +02:00
return $syncTool -> setupHttpSocketFeed ();
2018-07-19 11:48:22 +02:00
}
2016-06-04 01:08:16 +02:00
2019-08-25 12:23:33 +02:00
/**
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket
2019-08-25 12:23:33 +02:00
* @ param array $feed
* @ param string $uuid
2021-05-05 17:20:40 +02:00
* @ param array $user
2019-08-28 17:09:28 +02:00
* @ param array | bool $filterRules
2019-08-25 12:23:33 +02:00
* @ return array | bool | string
* @ throws Exception
*/
2020-07-28 17:37:36 +02:00
private function __addEventFromFeed ( HttpSocket $HttpSocket = null , $feed , $uuid , $user , $filterRules )
2018-07-19 11:48:22 +02:00
{
2019-08-25 12:23:33 +02:00
$event = $this -> downloadAndParseEventFromFeed ( $feed , $uuid , $HttpSocket );
2019-08-28 17:09:28 +02:00
$event = $this -> __prepareEvent ( $event , $feed , $filterRules );
2018-07-19 11:48:22 +02:00
if ( is_array ( $event )) {
return $this -> Event -> _add ( $event , true , $user );
} else {
return $event ;
}
}
2016-06-04 01:08:16 +02:00
2019-08-25 12:23:33 +02:00
/**
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2019-08-25 12:23:33 +02:00
* @ param array $feed
* @ param string $uuid
* @ param $user
2019-08-28 17:09:28 +02:00
* @ param array | bool $filterRules
2019-08-25 12:23:33 +02:00
* @ return mixed
* @ throws Exception
*/
2021-10-03 19:56:23 +02:00
private function __updateEventFromFeed ( HttpSocket $HttpSocket = null , $feed , $uuid , $user , $filterRules )
2018-07-19 11:48:22 +02:00
{
2019-08-25 12:23:33 +02:00
$event = $this -> downloadAndParseEventFromFeed ( $feed , $uuid , $HttpSocket );
2019-08-28 17:09:28 +02:00
$event = $this -> __prepareEvent ( $event , $feed , $filterRules );
2018-07-19 11:48:22 +02:00
return $this -> Event -> _edit ( $event , $user , $uuid , $jobId = null );
}
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
public function addDefaultFeeds ( $newFeeds )
{
foreach ( $newFeeds as $newFeed ) {
$existingFeed = $this -> find ( 'list' , array ( 'conditions' => array ( 'Feed.url' => $newFeed [ 'url' ])));
$success = true ;
if ( empty ( $existingFeed )) {
$this -> create ();
$feed = array (
2021-06-24 12:07:09 +02:00
'name' => $newFeed [ 'name' ],
'provider' => $newFeed [ 'provider' ],
'url' => $newFeed [ 'url' ],
'enabled' => $newFeed [ 'enabled' ],
'caching_enabled' => ! empty ( $newFeed [ 'caching_enabled' ]) ? $newFeed [ 'caching_enabled' ] : 0 ,
'distribution' => 3 ,
'sharing_group_id' => 0 ,
'tag_id' => 0 ,
'default' => true ,
2018-07-19 11:48:22 +02:00
);
$result = $this -> save ( $feed ) && $success ;
}
}
return $success ;
}
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
public function downloadFromFeedInitiator ( $feedId , $user , $jobId = false )
{
$this -> id = $feedId ;
$this -> read ();
if ( isset ( $this -> data [ 'Feed' ][ 'settings' ]) && ! empty ( $this -> data [ 'Feed' ][ 'settings' ])) {
$this -> data [ 'Feed' ][ 'settings' ] = json_decode ( $this -> data [ 'Feed' ][ 'settings' ], true );
}
2019-08-26 19:30:53 +02:00
2020-07-28 17:37:36 +02:00
$HttpSocket = $this -> isFeedLocal ( $this -> data ) ? null : $this -> __setupHttpSocket ( $this -> data );
2021-05-05 17:20:40 +02:00
if ( $this -> data [ 'Feed' ][ 'source_format' ] === 'misp' ) {
2019-08-26 19:47:25 +02:00
$this -> jobProgress ( $jobId , 'Fetching event manifest.' );
2019-08-25 10:17:50 +02:00
try {
$actions = $this -> getNewEventUuids ( $this -> data , $HttpSocket );
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not get new event uuids for feed $feedId . " , $e );
2021-05-05 17:20:40 +02:00
$this -> jobProgress ( $jobId , 'Could not fetch event manifest. See error log for more details.' );
2019-08-25 10:17:50 +02:00
return false ;
2018-07-19 11:48:22 +02:00
}
2019-08-26 19:47:25 +02:00
2019-08-26 09:15:25 +02:00
if ( empty ( $actions [ 'add' ]) && empty ( $actions [ 'edit' ])) {
2018-07-19 11:48:22 +02:00
return true ;
}
2019-08-26 19:47:25 +02:00
$total = count ( $actions [ 'add' ]) + count ( $actions [ 'edit' ]);
2021-05-05 17:20:40 +02:00
$this -> jobProgress ( $jobId , __ ( " Fetching %s events. " , $total ));
2018-07-19 11:48:22 +02:00
$result = $this -> downloadFromFeed ( $actions , $this -> data , $HttpSocket , $user , $jobId );
$this -> __cleanupFile ( $this -> data , '/manifest.json' );
} else {
2019-08-26 19:47:25 +02:00
$this -> jobProgress ( $jobId , 'Fetching data.' );
2019-08-25 11:22:36 +02:00
try {
$temp = $this -> getFreetextFeed ( $this -> data , $HttpSocket , $this -> data [ 'Feed' ][ 'source_format' ], 'all' );
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not get freetext feed $feedId " , $e );
2020-07-28 17:37:36 +02:00
$this -> jobProgress ( $jobId , 'Could not fetch freetext feed. See error log for more details.' );
2019-08-25 11:22:36 +02:00
return false ;
}
2019-08-26 19:47:25 +02:00
2020-05-11 17:06:55 +02:00
if ( empty ( $temp )) {
return true ;
}
2019-08-25 11:22:36 +02:00
$data = array ();
2019-09-03 17:24:06 +02:00
foreach ( $temp as $value ) {
2019-08-25 11:22:36 +02:00
$data [] = array (
'category' => $value [ 'category' ],
'type' => $value [ 'default_type' ],
'value' => $value [ 'value' ],
'to_ids' => $value [ 'to_ids' ]
);
2018-11-23 14:11:33 +01:00
}
2019-09-03 17:24:06 +02:00
$this -> jobProgress ( $jobId , 'Saving data.' , 50 );
try {
2020-05-11 17:06:55 +02:00
$result = $this -> saveFreetextFeedData ( $this -> data , $data , $user , $jobId );
2019-09-03 17:24:06 +02:00
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not save freetext feed data for feed $feedId . " , $e );
2018-07-19 11:48:22 +02:00
return false ;
}
2019-09-03 17:24:06 +02:00
2018-07-19 11:48:22 +02:00
$this -> __cleanupFile ( $this -> data , '' );
}
return $result ;
}
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
private function __cleanupFile ( $feed , $file )
{
2019-08-26 19:30:53 +02:00
if ( $this -> isFeedLocal ( $feed )) {
2018-07-19 11:48:22 +02:00
if ( isset ( $feed [ 'Feed' ][ 'delete_local_file' ]) && $feed [ 'Feed' ][ 'delete_local_file' ]) {
2021-10-04 09:45:58 +02:00
FileAccessTool :: deleteFileIfExists ( $feed [ 'Feed' ][ 'url' ] . $file );
2018-07-19 11:48:22 +02:00
}
}
return true ;
}
2016-12-30 16:16:56 +01:00
2019-09-03 17:24:06 +02:00
/**
* @ param array $feed
* @ param array $data
* @ param array $user
* @ param int | bool $jobId
* @ return bool
* @ throws Exception
*/
2021-02-16 16:10:25 +01:00
public function saveFreetextFeedData ( array $feed , array $data , array $user , $jobId = false )
2018-07-19 11:48:22 +02:00
{
$this -> Event = ClassRegistry :: init ( 'Event' );
2019-09-03 17:24:06 +02:00
2018-07-19 11:48:22 +02:00
if ( $feed [ 'Feed' ][ 'fixed_event' ] && $feed [ 'Feed' ][ 'event_id' ]) {
$event = $this -> Event -> find ( 'first' , array ( 'conditions' => array ( 'Event.id' => $feed [ 'Feed' ][ 'event_id' ]), 'recursive' => - 1 ));
if ( empty ( $event )) {
2019-09-03 17:24:06 +02:00
throw new Exception ( " The target event is no longer valid. Make sure that the target event { $feed [ 'Feed' ][ 'event_id' ] } exists. " );
2018-07-19 11:48:22 +02:00
}
2019-09-03 17:24:06 +02:00
} else {
2018-07-19 11:48:22 +02:00
$this -> Event -> create ();
2020-04-17 14:17:54 +02:00
$orgc_id = $user [ 'org_id' ];
if ( ! empty ( $feed [ 'Feed' ][ 'orgc_id' ])) {
$orgc_id = $feed [ 'Feed' ][ 'orgc_id' ];
}
2018-07-19 11:48:22 +02:00
$event = array (
2021-06-24 12:07:09 +02:00
'info' => $feed [ 'Feed' ][ 'name' ] . ' feed' ,
'analysis' => 2 ,
'threat_level_id' => 4 ,
'orgc_id' => $orgc_id ,
'org_id' => $user [ 'org_id' ],
'date' => date ( 'Y-m-d' ),
'distribution' => $feed [ 'Feed' ][ 'distribution' ],
'sharing_group_id' => $feed [ 'Feed' ][ 'sharing_group_id' ],
'user_id' => $user [ 'id' ]
2018-07-19 11:48:22 +02:00
);
$result = $this -> Event -> save ( $event );
if ( ! $result ) {
2019-09-03 17:24:06 +02:00
throw new Exception ( 'Something went wrong while creating a new event.' );
2018-07-19 11:48:22 +02:00
}
$event = $this -> Event -> find ( 'first' , array ( 'conditions' => array ( 'Event.id' => $this -> Event -> id ), 'recursive' => - 1 ));
if ( empty ( $event )) {
2019-09-03 17:24:06 +02:00
throw new Exception ( " The newly created event is no longer valid. Make sure that the target event { $this -> Event -> id } exists. " );
2018-07-19 11:48:22 +02:00
}
if ( $feed [ 'Feed' ][ 'fixed_event' ]) {
$feed [ 'Feed' ][ 'event_id' ] = $event [ 'Event' ][ 'id' ];
if ( ! empty ( $feed [ 'Feed' ][ 'settings' ])) {
$feed [ 'Feed' ][ 'settings' ] = json_encode ( $feed [ 'Feed' ][ 'settings' ]);
}
$this -> save ( $feed );
}
}
if ( $feed [ 'Feed' ][ 'fixed_event' ]) {
2021-02-16 16:10:25 +01:00
$existsAttributesValueToId = $this -> Event -> Attribute -> find ( 'list' , array (
2018-07-19 11:48:22 +02:00
'conditions' => array (
'Attribute.deleted' => 0 ,
'Attribute.event_id' => $event [ 'Event' ][ 'id' ]
),
'recursive' => - 1 ,
2021-02-16 16:10:25 +01:00
'fields' => array ( 'value' , 'id' )
2018-07-19 11:48:22 +02:00
));
2020-05-11 17:06:55 +02:00
// Create event diff. After this cycle, `$data` will contains just attributes that do not exists in current
// event and in `$existsAttributesValueToId` will contains just attributes that do not exists in current feed.
2018-07-19 11:48:22 +02:00
foreach ( $data as $k => $dataPoint ) {
2020-05-11 17:06:55 +02:00
if ( isset ( $existsAttributesValueToId [ $dataPoint [ 'value' ]])) {
2018-07-19 11:48:22 +02:00
unset ( $data [ $k ]);
2020-05-11 17:06:55 +02:00
unset ( $existsAttributesValueToId [ $dataPoint [ 'value' ]]);
2020-05-12 11:42:55 +02:00
continue ;
2018-07-19 11:48:22 +02:00
}
2020-05-12 11:42:55 +02:00
// Because some types can be saved in modified version (for example, IPv6 address is convert to compressed
// format, we should also check if current event contains modified value.
$modifiedValue = $this -> Event -> Attribute -> modifyBeforeValidation ( $dataPoint [ 'type' ], $dataPoint [ 'value' ]);
if ( isset ( $existsAttributesValueToId [ $modifiedValue ])) {
2018-07-19 11:48:22 +02:00
unset ( $data [ $k ]);
2020-05-12 11:42:55 +02:00
unset ( $existsAttributesValueToId [ $modifiedValue ]);
2018-07-19 11:48:22 +02:00
}
}
2020-05-11 17:06:55 +02:00
if ( $feed [ 'Feed' ][ 'delta_merge' ] && ! empty ( $existsAttributesValueToId )) {
2020-06-03 16:17:16 +02:00
$attributesToDelete = $this -> Event -> Attribute -> find ( 'all' , array (
'conditions' => array (
'Attribute.id' => array_values ( $existsAttributesValueToId )
),
'recursive' => - 1
));
foreach ( $attributesToDelete as $k => $attribute ) {
$attributesToDelete [ $k ][ 'Attribute' ][ 'deleted' ] = 1 ;
2020-07-21 11:50:31 +02:00
unset ( $attributesToDelete [ $k ][ 'Attribute' ][ 'timestamp' ]);
2020-06-03 16:17:16 +02:00
}
$this -> Event -> Attribute -> saveMany ( $attributesToDelete ); // We need to trigger callback methods
2020-07-21 11:50:31 +02:00
if ( ! empty ( $attributesToDelete )) {
$this -> Event -> unpublishEvent ( $feed [ 'Feed' ][ 'event_id' ]);
}
2018-07-19 11:48:22 +02:00
}
}
2020-07-21 11:50:31 +02:00
if ( empty ( $data ) && empty ( $attributesToDelete )) {
2018-07-19 11:48:22 +02:00
return true ;
}
2019-09-03 17:24:06 +02:00
2018-07-19 11:48:22 +02:00
$uniqueValues = array ();
foreach ( $data as $key => $value ) {
2019-09-03 17:24:06 +02:00
if ( isset ( $uniqueValues [ $value [ 'value' ]])) {
2018-07-19 11:48:22 +02:00
unset ( $data [ $key ]);
continue ;
}
$data [ $key ][ 'event_id' ] = $event [ 'Event' ][ 'id' ];
$data [ $key ][ 'distribution' ] = $feed [ 'Feed' ][ 'distribution' ];
$data [ $key ][ 'sharing_group_id' ] = $feed [ 'Feed' ][ 'sharing_group_id' ];
2019-09-03 17:24:06 +02:00
$data [ $key ][ 'to_ids' ] = $feed [ 'Feed' ][ 'override_ids' ] ? 0 : $value [ 'to_ids' ];
$uniqueValues [ $value [ 'value' ]] = true ;
2018-07-19 11:48:22 +02:00
}
2021-10-03 17:56:30 +02:00
$chunks = array_chunk ( $data , 100 );
foreach ( $chunks as $k => $chunk ) {
$this -> Event -> Attribute -> saveMany ( $chunk , [ 'validate' => true , 'parentEvent' => $event ]);
$this -> jobProgress ( $jobId , null , 50 + round (( $k * 100 + 1 ) / count ( $data ) * 50 ));
2018-07-19 11:48:22 +02:00
}
2020-07-21 11:50:31 +02:00
if ( ! empty ( $data ) || ! empty ( $attributesToDelete )) {
2019-02-27 20:53:33 +01:00
unset ( $event [ 'Event' ][ 'timestamp' ]);
2019-04-05 10:46:32 +02:00
unset ( $event [ 'Event' ][ 'attribute_count' ]);
2019-02-27 20:53:33 +01:00
$this -> Event -> save ( $event );
}
2018-07-19 11:48:22 +02:00
if ( $feed [ 'Feed' ][ 'publish' ]) {
$this -> Event -> publishRouter ( $event [ 'Event' ][ 'id' ], null , $user );
}
if ( $feed [ 'Feed' ][ 'tag_id' ]) {
2021-03-25 16:04:23 +01:00
$this -> Event -> EventTag -> attachTagToEvent ( $event [ 'Event' ][ 'id' ], [ 'id' => $feed [ 'Feed' ][ 'tag_id' ]]);
2018-07-19 11:48:22 +02:00
}
return true ;
}
2017-01-25 05:56:32 +01:00
2019-08-28 17:01:07 +02:00
/**
2019-10-02 18:55:37 +02:00
* @ param $user - Not used
2019-08-28 17:01:07 +02:00
* @ param int | bool $jobId
* @ param string $scope
2019-10-02 18:55:37 +02:00
* @ return array
2019-08-28 17:01:07 +02:00
* @ throws Exception
*/
2018-07-19 11:48:22 +02:00
public function cacheFeedInitiator ( $user , $jobId = false , $scope = 'freetext' )
{
$params = array (
'conditions' => array ( 'caching_enabled' => 1 ),
'recursive' => - 1 ,
'fields' => array ( 'source_format' , 'input_source' , 'url' , 'id' , 'settings' , 'headers' )
);
2019-10-02 18:55:37 +02:00
$redis = $this -> setupRedisWithException ();
2018-07-19 11:48:22 +02:00
if ( $scope !== 'all' ) {
if ( is_numeric ( $scope )) {
$params [ 'conditions' ][ 'id' ] = $scope ;
} elseif ( $scope == 'freetext' || $scope == 'csv' ) {
$params [ 'conditions' ][ 'source_format' ] = array ( 'csv' , 'freetext' );
} elseif ( $scope == 'misp' ) {
2019-09-15 13:05:55 +02:00
$redis -> del ( $redis -> keys ( 'misp:feed_cache:event_uuid_lookup:*' ));
2018-07-19 11:48:22 +02:00
$params [ 'conditions' ][ 'source_format' ] = 'misp' ;
2019-08-28 17:01:07 +02:00
} else {
throw new InvalidArgumentException ( " Invalid value for scope, it must be integer or 'freetext', 'csv', 'misp' or 'all' string. " );
2018-07-19 11:48:22 +02:00
}
} else {
$redis -> del ( 'misp:feed_cache:combined' );
2019-09-15 13:05:55 +02:00
$redis -> del ( $redis -> keys ( 'misp:feed_cache:event_uuid_lookup:*' ));
2018-07-19 11:48:22 +02:00
}
$feeds = $this -> find ( 'all' , $params );
2019-10-02 18:55:37 +02:00
$results = array ( 'successes' => 0 , 'fails' => 0 );
2018-07-19 11:48:22 +02:00
foreach ( $feeds as $k => $feed ) {
2019-08-26 19:47:25 +02:00
if ( $this -> __cacheFeed ( $feed , $redis , $jobId )) {
$message = 'Feed ' . $feed [ 'Feed' ][ 'id' ] . ' cached.' ;
2019-10-02 18:55:37 +02:00
$results [ 'successes' ] ++ ;
2019-08-26 19:47:25 +02:00
} else {
$message = 'Failed to cache feed ' . $feed [ 'Feed' ][ 'id' ] . '. See logs for more details.' ;
2019-10-02 18:55:37 +02:00
$results [ 'fails' ] ++ ;
2018-07-19 11:48:22 +02:00
}
2019-08-26 19:47:25 +02:00
$this -> jobProgress ( $jobId , $message , 100 * $k / count ( $feeds ));
2018-07-19 11:48:22 +02:00
}
2019-10-02 18:55:37 +02:00
return $results ;
2018-07-19 11:48:22 +02:00
}
2017-05-08 14:22:27 +02:00
2020-09-28 17:19:25 +02:00
/**
* @ param array $feeds
* @ return array
*/
public function attachFeedCacheTimestamps ( array $feeds )
2018-07-19 11:48:22 +02:00
{
2020-09-28 17:19:25 +02:00
try {
$redis = $this -> setupRedisWithException ();
} catch ( Exception $e ) {
return $feeds ;
}
$pipe = $redis -> multi ( Redis :: PIPELINE );
foreach ( $feeds as $feed ) {
$pipe -> get ( 'misp:feed_cache_timestamp:' . $feed [ 'Feed' ][ 'id' ]);
2018-07-19 11:48:22 +02:00
}
2020-09-28 17:19:25 +02:00
$result = $redis -> exec ();
foreach ( $feeds as $k => $feed ) {
$feeds [ $k ][ 'Feed' ][ 'cache_timestamp' ] = $result [ $k ];
2018-07-19 11:48:22 +02:00
}
2020-09-28 17:19:25 +02:00
return $feeds ;
2018-07-19 11:48:22 +02:00
}
2017-05-08 14:22:27 +02:00
2018-07-19 11:48:22 +02:00
private function __cacheFeed ( $feed , $redis , $jobId = false )
{
2020-07-28 17:37:36 +02:00
$HttpSocket = $this -> isFeedLocal ( $feed ) ? null : $this -> __setupHttpSocket ( $feed );
2020-09-28 18:33:56 +02:00
if ( $feed [ 'Feed' ][ 'source_format' ] === 'misp' ) {
2018-07-19 11:48:22 +02:00
return $this -> __cacheMISPFeed ( $feed , $redis , $HttpSocket , $jobId );
} else {
return $this -> __cacheFreetextFeed ( $feed , $redis , $HttpSocket , $jobId );
}
}
2017-05-08 14:22:27 +02:00
2020-09-28 17:00:00 +02:00
/**
* @ param array $feed
* @ param Redis $redis
* @ param HttpSocket | null $HttpSocket
* @ param int | false $jobId
* @ return bool
*/
2020-07-28 17:37:36 +02:00
private function __cacheFreetextFeed ( array $feed , $redis , HttpSocket $HttpSocket = null , $jobId = false )
2018-07-19 11:48:22 +02:00
{
2019-08-26 21:43:41 +02:00
$feedId = $feed [ 'Feed' ][ 'id' ];
2020-09-28 17:00:00 +02:00
$this -> jobProgress ( $jobId , __ ( " Feed %s: Fetching. " , $feedId ));
2019-08-25 11:22:36 +02:00
try {
$values = $this -> getFreetextFeed ( $feed , $HttpSocket , $feed [ 'Feed' ][ 'source_format' ], 'all' );
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not get freetext feed $feedId " , $e );
2020-11-19 09:53:02 +01:00
$this -> jobProgress ( $jobId , __ ( 'Could not fetch freetext feed %s. See error log for more details.' , $feedId ));
2019-08-25 11:22:36 +02:00
return false ;
2018-07-19 11:48:22 +02:00
}
2019-08-25 11:22:36 +02:00
2020-11-19 09:53:02 +01:00
// Convert values to MD5 hashes
$md5Values = array_map ( 'md5' , array_column ( $values , 'value' ));
2020-09-28 18:39:43 +02:00
$redis -> del ( 'misp:feed_cache:' . $feedId );
2021-02-24 20:28:37 +01:00
foreach ( array_chunk ( $md5Values , 5000 ) as $k => $chunk ) {
2020-11-19 09:53:02 +01:00
$pipe = $redis -> multi ( Redis :: PIPELINE );
if ( method_exists ( $redis , 'sAddArray' )) {
$redis -> sAddArray ( 'misp:feed_cache:' . $feedId , $chunk );
$redis -> sAddArray ( 'misp:feed_cache:combined' , $chunk );
} else {
foreach ( $chunk as $value ) {
$redis -> sAdd ( 'misp:feed_cache:' . $feedId , $value );
$redis -> sAdd ( 'misp:feed_cache:combined' , $value );
}
2019-08-25 11:22:36 +02:00
}
2020-11-19 09:53:02 +01:00
$pipe -> exec ();
2021-02-24 20:28:37 +01:00
$this -> jobProgress ( $jobId , __ ( 'Feed %s: %s/%s values cached.' , $feedId , $k * 5000 , count ( $md5Values )));
2019-08-25 11:22:36 +02:00
}
2020-11-19 09:53:02 +01:00
$redis -> set ( 'misp:feed_cache_timestamp:' . $feedId , time ());
2019-08-25 11:22:36 +02:00
return true ;
2018-07-19 11:48:22 +02:00
}
2017-05-08 14:22:27 +02:00
2020-07-28 17:37:36 +02:00
private function __cacheMISPFeedTraditional ( $feed , $redis , HttpSocket $HttpSocket = null , $jobId = false )
2018-07-19 11:48:22 +02:00
{
2019-08-26 21:43:41 +02:00
$feedId = $feed [ 'Feed' ][ 'id' ];
2019-08-25 10:17:50 +02:00
try {
$manifest = $this -> getManifest ( $feed , $HttpSocket );
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not get manifest for feed $feedId . " , $e );
2018-07-19 11:48:22 +02:00
return false ;
}
2019-08-25 10:17:50 +02:00
2019-08-26 21:43:41 +02:00
$redis -> del ( 'misp:feed_cache:' . $feedId );
2019-08-25 10:17:50 +02:00
2018-07-19 11:48:22 +02:00
$k = 0 ;
foreach ( $manifest as $uuid => $event ) {
2019-08-25 12:23:33 +02:00
try {
$event = $this -> downloadAndParseEventFromFeed ( $feed , $uuid , $HttpSocket );
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not get and parse event ' $uuid ' for feed $feedId . " , $e );
2019-08-25 12:23:33 +02:00
return false ;
2018-07-19 11:48:22 +02:00
}
2019-08-25 12:23:33 +02:00
if ( ! empty ( $event [ 'Event' ][ 'Attribute' ])) {
2020-07-28 17:37:36 +02:00
$this -> Attribute = ClassRegistry :: init ( 'Attribute' );
2019-08-25 12:23:33 +02:00
$pipe = $redis -> multi ( Redis :: PIPELINE );
foreach ( $event [ 'Event' ][ 'Attribute' ] as $attribute ) {
2021-07-27 15:19:41 +02:00
if ( ! in_array ( $attribute [ 'type' ], Attribute :: NON_CORRELATING_TYPES , true )) {
2019-08-25 12:23:33 +02:00
if ( in_array ( $attribute [ 'type' ], $this -> Attribute -> getCompositeTypes ())) {
$value = explode ( '|' , $attribute [ 'value' ]);
2021-07-21 08:42:05 +02:00
if ( in_array ( $attribute [ 'type' ], Attribute :: PRIMARY_ONLY_CORRELATING_TYPES , true )) {
2020-09-28 12:24:35 +02:00
unset ( $value [ 1 ]);
}
2019-08-25 12:23:33 +02:00
} else {
2020-09-28 12:24:35 +02:00
$value = [ $attribute [ 'value' ]];
}
foreach ( $value as $v ) {
$md5 = md5 ( $v );
$redis -> sAdd ( 'misp:feed_cache:' . $feedId , $md5 );
$redis -> sAdd ( 'misp:feed_cache:combined' , $md5 );
$redis -> sAdd ( 'misp:feed_cache:event_uuid_lookup:' . $md5 , $feedId . '/' . $event [ 'Event' ][ 'uuid' ]);
2018-07-19 11:48:22 +02:00
}
}
}
2019-08-25 12:23:33 +02:00
$pipe -> exec ();
2018-07-19 11:48:22 +02:00
}
2019-08-25 12:23:33 +02:00
2018-07-19 11:48:22 +02:00
$k ++ ;
2019-08-26 19:47:25 +02:00
if ( $k % 10 == 0 ) {
2019-08-26 21:43:41 +02:00
$this -> jobProgress ( $jobId , " Feed $feedId : $k / " . count ( $manifest ) . " events cached. " );
2018-07-19 11:48:22 +02:00
}
}
return true ;
}
2017-05-08 14:22:27 +02:00
2020-07-28 17:37:36 +02:00
private function __cacheMISPFeedCache ( $feed , $redis , HttpSocket $HttpSocket = null , $jobId = false )
2018-07-19 11:48:22 +02:00
{
2019-08-26 21:43:41 +02:00
$feedId = $feed [ 'Feed' ][ 'id' ];
2019-08-26 09:01:56 +02:00
try {
$cache = $this -> getCache ( $feed , $HttpSocket );
} catch ( Exception $e ) {
2019-09-24 20:54:11 +02:00
$this -> logException ( " Could not get cache file for $feedId . " , $e , LOG_NOTICE );
2018-07-19 11:48:22 +02:00
return false ;
}
2019-08-26 09:01:56 +02:00
2018-07-19 11:48:22 +02:00
$pipe = $redis -> multi ( Redis :: PIPELINE );
2020-09-28 18:39:43 +02:00
$redis -> del ( 'misp:feed_cache:' . $feedId );
2019-08-26 09:01:56 +02:00
foreach ( $cache as $v ) {
2020-09-28 12:24:35 +02:00
list ( $hash , $eventUuid ) = $v ;
$redis -> sAdd ( 'misp:feed_cache:' . $feedId , $hash );
$redis -> sAdd ( 'misp:feed_cache:combined' , $hash );
$redis -> sAdd ( 'misp:feed_cache:event_uuid_lookup:' . $hash , " $feedId / $eventUuid " );
2018-07-19 11:48:22 +02:00
}
$pipe -> exec ();
2019-08-26 21:43:41 +02:00
$this -> jobProgress ( $jobId , " Feed $feedId : cached via quick cache. " );
2018-07-19 11:48:22 +02:00
return true ;
}
2017-12-22 14:47:00 +01:00
2020-07-28 17:37:36 +02:00
private function __cacheMISPFeed ( $feed , $redis , HttpSocket $HttpSocket = null , $jobId = false )
2018-07-19 11:48:22 +02:00
{
2019-08-26 19:47:25 +02:00
$result = true ;
2018-07-19 11:48:22 +02:00
if ( ! $this -> __cacheMISPFeedCache ( $feed , $redis , $HttpSocket , $jobId )) {
2019-08-26 19:47:25 +02:00
$result = $this -> __cacheMISPFeedTraditional ( $feed , $redis , $HttpSocket , $jobId );
2020-09-28 18:33:56 +02:00
}
2019-08-26 19:47:25 +02:00
if ( $result ) {
$redis -> set ( 'misp:feed_cache_timestamp:' . $feed [ 'Feed' ][ 'id' ], time ());
}
return $result ;
2018-07-19 11:48:22 +02:00
}
2017-12-22 14:47:00 +01:00
2018-07-19 11:48:22 +02:00
public function compareFeeds ( $id = false )
{
$redis = $this -> setupRedis ();
if ( $redis === false ) {
return array ();
}
$fields = array ( 'id' , 'input_source' , 'source_format' , 'url' , 'provider' , 'name' , 'default' );
$feeds = $this -> find ( 'all' , array (
'recursive' => - 1 ,
2018-09-07 13:49:56 +02:00
'fields' => $fields ,
2018-11-23 14:11:33 +01:00
'conditions' => array ( 'Feed.caching_enabled' => 1 )
2018-07-19 11:48:22 +02:00
));
// we'll use this later for the intersect
$fields [] = 'values' ;
$fields = array_flip ( $fields );
// Get all of the feed cache cardinalities for all feeds - if a feed is not cached remove it from the list
foreach ( $feeds as $k => $feed ) {
if ( ! $redis -> exists ( 'misp:feed_cache:' . $feed [ 'Feed' ][ 'id' ])) {
unset ( $feeds [ $k ]);
continue ;
}
$feeds [ $k ][ 'Feed' ][ 'values' ] = $redis -> sCard ( 'misp:feed_cache:' . $feed [ 'Feed' ][ 'id' ]);
}
$feeds = array_values ( $feeds );
2019-01-20 10:19:05 +01:00
$this -> Server = ClassRegistry :: init ( 'Server' );
$servers = $this -> Server -> find ( 'all' , array (
'recursive' => - 1 ,
'fields' => array ( 'id' , 'url' , 'name' ),
'contain' => array ( 'RemoteOrg' => array ( 'fields' => array ( 'RemoteOrg.id' , 'RemoteOrg.name' ))),
2020-03-30 15:19:43 +02:00
'conditions' => array ( 'Server.caching_enabled' => 1 )
2019-01-20 10:19:05 +01:00
));
foreach ( $servers as $k => $server ) {
if ( ! $redis -> exists ( 'misp:server_cache:' . $server [ 'Server' ][ 'id' ])) {
unset ( $servers [ $k ]);
continue ;
}
$servers [ $k ][ 'Server' ][ 'input_source' ] = 'network' ;
$servers [ $k ][ 'Server' ][ 'source_format' ] = 'misp' ;
$servers [ $k ][ 'Server' ][ 'provider' ] = $servers [ $k ][ 'RemoteOrg' ][ 'name' ];
$servers [ $k ][ 'Server' ][ 'default' ] = false ;
$servers [ $k ][ 'Server' ][ 'is_misp_server' ] = true ;
$servers [ $k ][ 'Server' ][ 'values' ] = $redis -> sCard ( 'misp:server_cache:' . $server [ 'Server' ][ 'id' ]);
}
2018-07-19 11:48:22 +02:00
foreach ( $feeds as $k => $feed ) {
foreach ( $feeds as $k2 => $feed2 ) {
if ( $k == $k2 ) {
continue ;
}
$intersect = $redis -> sInter ( 'misp:feed_cache:' . $feed [ 'Feed' ][ 'id' ], 'misp:feed_cache:' . $feed2 [ 'Feed' ][ 'id' ]);
$feeds [ $k ][ 'Feed' ][ 'ComparedFeed' ][] = array_merge ( array_intersect_key ( $feed2 [ 'Feed' ], $fields ), array (
'overlap_count' => count ( $intersect ),
'overlap_percentage' => round ( 100 * count ( $intersect ) / $feeds [ $k ][ 'Feed' ][ 'values' ]),
));
}
2019-01-20 10:19:05 +01:00
foreach ( $servers as $k2 => $server ) {
$intersect = $redis -> sInter ( 'misp:feed_cache:' . $feed [ 'Feed' ][ 'id' ], 'misp:server_cache:' . $server [ 'Server' ][ 'id' ]);
$feeds [ $k ][ 'Feed' ][ 'ComparedFeed' ][] = array_merge ( array_intersect_key ( $server [ 'Server' ], $fields ), array (
'overlap_count' => count ( $intersect ),
'overlap_percentage' => round ( 100 * count ( $intersect ) / $feeds [ $k ][ 'Feed' ][ 'values' ]),
));
}
}
foreach ( $servers as $k => $server ) {
foreach ( $feeds as $k2 => $feed2 ) {
$intersect = $redis -> sInter ( 'misp:server_cache:' . $server [ 'Server' ][ 'id' ], 'misp:feed_cache:' . $feed2 [ 'Feed' ][ 'id' ]);
$servers [ $k ][ 'Server' ][ 'ComparedFeed' ][] = array_merge ( array_intersect_key ( $feed2 [ 'Feed' ], $fields ), array (
'overlap_count' => count ( $intersect ),
'overlap_percentage' => round ( 100 * count ( $intersect ) / $servers [ $k ][ 'Server' ][ 'values' ]),
));
}
foreach ( $servers as $k2 => $server2 ) {
if ( $k == $k2 ) {
continue ;
}
$intersect = $redis -> sInter ( 'misp:server_cache:' . $server [ 'Server' ][ 'id' ], 'misp:server_cache:' . $server2 [ 'Server' ][ 'id' ]);
$servers [ $k ][ 'Server' ][ 'ComparedFeed' ][] = array_merge ( array_intersect_key ( $server2 [ 'Server' ], $fields ), array (
'overlap_count' => count ( $intersect ),
'overlap_percentage' => round ( 100 * count ( $intersect ) / $servers [ $k ][ 'Server' ][ 'values' ]),
));
}
}
foreach ( $servers as $k => $server ) {
$server [ 'Feed' ] = $server [ 'Server' ];
unset ( $server [ 'Server' ]);
$feeds [] = $server ;
2018-07-19 11:48:22 +02:00
}
return $feeds ;
}
2017-05-08 14:22:27 +02:00
2018-07-19 11:48:22 +02:00
public function importFeeds ( $feeds , $user , $default = false )
{
2019-10-02 16:41:03 +02:00
if ( is_string ( $feeds )) {
$feeds = json_decode ( $feeds , true );
}
2019-10-02 16:37:52 +02:00
if ( $feeds && ! isset ( $feeds [ 0 ])) {
2018-07-19 11:48:22 +02:00
$feeds = array ( $feeds );
}
$results = array ( 'successes' => 0 , 'fails' => 0 );
if ( empty ( $feeds )) {
return $results ;
}
$existingFeeds = $this -> find ( 'all' , array ());
foreach ( $feeds as $feed ) {
if ( $default ) {
$feed [ 'Feed' ][ 'default' ] = 1 ;
} else {
$feed [ 'Feed' ][ 'default' ] = 0 ;
}
if ( isset ( $feed [ 'Feed' ][ 'id' ])) {
unset ( $feed [ 'Feed' ][ 'id' ]);
}
$found = false ;
foreach ( $existingFeeds as $existingFeed ) {
if ( $existingFeed [ 'Feed' ][ 'url' ] == $feed [ 'Feed' ][ 'url' ]) {
$found = true ;
}
}
if ( ! $found ) {
$feed [ 'Feed' ][ 'tag_id' ] = 0 ;
if ( isset ( $feed [ 'Tag' ])) {
$tag_id = $this -> Tag -> captureTag ( $feed [ 'Tag' ], $user );
if ( $tag_id ) {
$feed [ 'Feed' ][ 'tag_id' ] = $tag_id ;
}
}
$this -> create ();
2021-02-16 13:08:29 +01:00
if ( ! $this -> save ( $feed , true , array ( 'name' , 'provider' , 'url' , 'rules' , 'source_format' , 'fixed_event' , 'delta_merge' , 'override_ids' , 'publish' , 'settings' , 'tag_id' , 'default' , 'lookup_visible' , 'headers' ))) {
2018-07-19 11:48:22 +02:00
$results [ 'fails' ] ++ ;
} else {
$results [ 'successes' ] ++ ;
}
}
}
return $results ;
}
2017-05-30 11:42:57 +02:00
2018-07-19 11:48:22 +02:00
public function load_default_feeds ()
{
$user = array ( 'Role' => array ( 'perm_tag_editor' => 1 , 'perm_site_admin' => 1 ));
$json = file_get_contents ( APP . 'files/feed-metadata/defaults.json' );
$this -> importFeeds ( $json , $user , true );
return true ;
}
2017-05-30 11:42:57 +02:00
2018-07-19 11:48:22 +02:00
public function setEnableFeedCachingDefaults ()
{
$feeds = $this -> find ( 'all' , array (
'conditions' => array (
'Feed.enabled' => 1
),
'recursive' => - 1
));
if ( empty ( $feeds )) {
return true ;
}
foreach ( $feeds as $feed ) {
$feed [ 'Feed' ][ 'caching_enabled' ] = 1 ;
$this -> save ( $feed );
}
return true ;
}
2019-03-09 22:19:37 +01:00
public function getFeedCoverage ( $id , $source_scope = 'feed' , $dataset = 'all' )
{
$redis = $this -> setupRedis ();
if ( $redis === false ) {
return 'Could not reach Redis.' ;
}
$this -> Server = ClassRegistry :: init ( 'Server' );
$feed_conditions = array ( 'Feed.caching_enabled' => 1 );
$server_conditions = array ( 'Server.caching_enabled' => 1 );
if ( $source_scope === 'feed' ) {
$feed_conditions [ 'NOT' ] = array ( 'Feed.id' => $id );
} else {
$server_conditions [ 'NOT' ] = array ( 'Server.id' => $id );
}
if ( $dataset !== 'all' ) {
if ( empty ( $dataset [ 'Feed' ])) {
$feed_conditions [ 'OR' ] = array ( 'Feed.id' => - 1 );
} else {
$feed_conditions [ 'OR' ] = array ( 'Feed.id' => $dataset [ 'Feed' ]);
}
if ( empty ( $dataset [ 'Server' ])) {
$server_conditions [ 'OR' ] = array ( 'Server.id' => - 1 );
} else {
$server_conditions [ 'OR' ] = array ( 'Server.id' => $dataset [ 'Server' ]);
}
}
$other_feeds = $this -> find ( 'list' , array (
'recursive' => - 1 ,
'conditions' => $feed_conditions ,
'fields' => array ( 'Feed.id' , 'Feed.id' )
));
$other_servers = $this -> Server -> find ( 'list' , array (
'recursive' => - 1 ,
'conditions' => $server_conditions ,
'fields' => array ( 'Server.id' , 'Server.id' )
));
$feed_element_count = $redis -> scard ( 'misp:feed_cache:' . $id );
$temp_store = ( new RandomTool ()) -> random_str ( false , 12 );
$params = array ( 'misp:feed_temp:' . $temp_store );
foreach ( $other_feeds as $other_feed ) {
$params [] = 'misp:feed_cache:' . $other_feed ;
}
foreach ( $other_servers as $other_server ) {
$params [] = 'misp:server_cache:' . $other_server ;
}
if ( count ( $params ) != 1 && $feed_element_count > 0 ) {
call_user_func_array ( array ( $redis , 'sunionstore' ), $params );
call_user_func_array ( array ( $redis , 'sinterstore' ), array ( 'misp:feed_temp:' . $temp_store . '_intersect' , 'misp:feed_cache:' . $id , 'misp:feed_temp:' . $temp_store ));
$cardinality_intersect = $redis -> scard ( 'misp:feed_temp:' . $temp_store . '_intersect' );
$coverage = round ( 100 * $cardinality_intersect / $feed_element_count , 2 );
$redis -> del ( 'misp:feed_temp:' . $temp_store );
$redis -> del ( 'misp:feed_temp:' . $temp_store . '_intersect' );
} else {
$coverage = 0 ;
}
return $coverage ;
}
2019-03-10 18:09:46 +01:00
public function getCachedElements ( $feedId )
{
$redis = $this -> setupRedis ();
$cardinality = $redis -> sCard ( 'misp:feed_cache:' . $feedId );
return $cardinality ;
}
2021-06-24 12:07:09 +02:00
public function getAllCachingEnabledFeeds ( $feedId , $intersectingOnly = false )
{
2019-03-10 18:09:46 +01:00
if ( $intersectingOnly ) {
$redis = $this -> setupRedis ();
}
2019-03-09 22:19:37 +01:00
$result [ 'Feed' ] = $this -> find ( 'all' , array (
'conditions' => array (
'Feed.id !=' => $feedId ,
'caching_enabled' => 1
),
'recursive' => - 1 ,
'fields' => array ( 'Feed.id' , 'Feed.name' , 'Feed.url' )
));
$this -> Server = ClassRegistry :: init ( 'Server' );
$result [ 'Server' ] = $this -> Server -> find ( 'all' , array (
'conditions' => array (
'caching_enabled' => 1
),
'recursive' => - 1 ,
'fields' => array ( 'Server.id' , 'Server.name' , 'Server.url' )
));
2019-03-10 18:09:46 +01:00
$scopes = array ( 'Feed' , 'Server' );
foreach ( $scopes as $scope ) {
foreach ( $result [ $scope ] as $k => $v ) {
$result [ $scope ][ $k ] = $v [ $scope ];
}
}
if ( $intersectingOnly ) {
foreach ( $scopes as $scope ) {
if ( ! empty ( $result [ $scope ])) {
foreach ( $result [ $scope ] as $k => $feed ) {
$intersect = $redis -> sInter ( 'misp:feed_cache:' . $feedId , 'misp:' . lcfirst ( $scope ) . '_cache:' . $feed [ 'id' ]);
if ( empty ( $intersect )) {
unset ( $result [ $scope ][ $k ]);
} else {
$result [ $scope ][ $k ][ 'matching_values' ] = count ( $intersect );
}
}
}
}
}
2019-03-09 22:19:37 +01:00
return $result ;
}
2019-04-01 16:09:24 +02:00
public function searchCaches ( $value )
{
$hits = array ();
$this -> Server = ClassRegistry :: init ( 'Server' );
$result [ 'Server' ] = $this -> Server -> find ( 'all' , array (
'conditions' => array (
'caching_enabled' => 1
),
'recursive' => - 1 ,
'fields' => array ( 'Server.id' , 'Server.name' , 'Server.url' )
));
$redis = $this -> setupRedis ();
2021-04-20 17:21:18 +02:00
$is_array = true ;
if ( ! is_array ( $value )) {
$is_array = false ;
if ( empty ( $value )) {
// old behaviour allowd for empty values to return all data
$value = [ false ];
} else {
$value = [ $value ];
}
}
foreach ( $value as $v ) {
if ( $v !== false ) {
$v = strtolower ( trim ( $v ));
}
if ( $v === false || $redis -> sismember ( 'misp:feed_cache:combined' , md5 ( $v ))) {
$feeds = $this -> find ( 'all' , array (
'conditions' => array (
'caching_enabled' => 1
),
'recursive' => - 1 ,
'fields' => array ( 'Feed.id' , 'Feed.name' , 'Feed.url' , 'Feed.source_format' )
));
foreach ( $feeds as $feed ) {
if (( $v === false ) || $redis -> sismember ( 'misp:feed_cache:' . $feed [ 'Feed' ][ 'id' ], md5 ( $v ))) {
if ( $feed [ 'Feed' ][ 'source_format' ] === 'misp' ) {
$uuid = $redis -> smembers ( 'misp:feed_cache:event_uuid_lookup:' . md5 ( $v ));
foreach ( $uuid as $k => $url ) {
$uuid [ $k ] = explode ( '/' , $url )[ 1 ];
}
$feed [ 'Feed' ][ 'uuid' ] = $uuid ;
if ( ! empty ( $feed [ 'Feed' ][ 'uuid' ])) {
foreach ( $feed [ 'Feed' ][ 'uuid' ] as $uuid ) {
$feed [ 'Feed' ][ 'direct_urls' ][] = array (
'url' => sprintf (
'%s/feeds/previewEvent/%s/%s' ,
Configure :: read ( 'MISP.baseurl' ),
h ( $feed [ 'Feed' ][ 'id' ]),
h ( $uuid )
),
'name' => __ ( 'Event %s' , $uuid )
);
}
}
$feed [ 'Feed' ][ 'type' ] = 'MISP Feed' ;
} else {
$feed [ 'Feed' ][ 'type' ] = 'Feed' ;
if ( ! empty ( $v )) {
2019-04-01 16:09:24 +02:00
$feed [ 'Feed' ][ 'direct_urls' ][] = array (
'url' => sprintf (
2021-04-20 17:21:18 +02:00
'%s/feeds/previewIndex/%s' ,
2019-04-01 16:09:24 +02:00
Configure :: read ( 'MISP.baseurl' ),
2021-04-20 17:21:18 +02:00
h ( $feed [ 'Feed' ][ 'id' ])
2019-04-01 16:09:24 +02:00
),
2021-04-20 17:21:18 +02:00
'name' => __ ( 'Feed %s' , $feed [ 'Feed' ][ 'id' ])
2019-04-01 16:09:24 +02:00
);
}
}
2021-04-20 17:21:18 +02:00
if ( $is_array ) {
$hits [ $v ][] = $feed ;
} else {
$hits [] = $feed ;
2019-04-01 16:09:24 +02:00
}
}
}
}
2021-04-20 17:21:18 +02:00
if ( $v === false || $redis -> sismember ( 'misp:server_cache:combined' , md5 ( $v ))) {
$this -> Server = ClassRegistry :: init ( 'Server' );
$servers = $this -> Server -> find ( 'all' , array (
'conditions' => array (
'caching_enabled' => 1
),
'recursive' => - 1 ,
'fields' => array ( 'Server.id' , 'Server.name' , 'Server.url' )
));
foreach ( $servers as $server ) {
if ( $v === false || $redis -> sismember ( 'misp:server_cache:' . $server [ 'Server' ][ 'id' ], md5 ( $v ))) {
$uuid = $redis -> smembers ( 'misp:server_cache:event_uuid_lookup:' . md5 ( $v ));
if ( ! empty ( $uuid )) {
foreach ( $uuid as $k => $url ) {
$uuid [ $k ] = explode ( '/' , $url )[ 1 ];
$server [ 'Server' ][ 'direct_urls' ][] = array (
'url' => sprintf (
'%s/servers/previewEvent/%s/%s' ,
Configure :: read ( 'MISP.baseurl' ),
h ( $server [ 'Server' ][ 'id' ]),
h ( $uuid [ $k ])
),
'name' => __ ( 'Event %s' , h ( $uuid [ $k ]))
);
}
}
$server [ 'Server' ][ 'uuid' ] = $uuid ;
$server [ 'Server' ][ 'type' ] = 'MISP Server' ;
if ( $is_array ) {
$hits [ $v ][] = array ( 'Feed' => $server [ 'Server' ]);
} else {
$hits [] = array ( 'Feed' => $server [ 'Server' ]);
2019-04-01 16:09:24 +02:00
}
}
}
}
}
return $hits ;
}
2019-08-25 12:23:33 +02:00
/**
* Download and parse event from feed .
2020-07-28 17:37:36 +02:00
*
2019-08-25 12:23:33 +02:00
* @ param array $feed
* @ param string $eventUuid
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2019-08-25 12:23:33 +02:00
* @ return array
* @ throws Exception
*/
2020-07-28 17:37:36 +02:00
private function downloadAndParseEventFromFeed ( $feed , $eventUuid , HttpSocket $HttpSocket = null )
2019-08-25 12:23:33 +02:00
{
if ( ! Validation :: uuid ( $eventUuid )) {
throw new InvalidArgumentException ( " Given event UUID ' $eventUuid ' is invalid. " );
}
$path = $feed [ 'Feed' ][ 'url' ] . '/' . $eventUuid . '.json' ;
2019-08-26 08:53:35 +02:00
$data = $this -> feedGetUri ( $feed , $path , $HttpSocket );
2020-07-28 18:33:05 +02:00
try {
return $this -> jsonDecode ( $data );
} catch ( Exception $e ) {
throw new Exception ( " Could not parse event JSON with UUID ' $eventUuid ' from feed " , 0 , $e );
}
2019-08-26 08:53:35 +02:00
}
/**
* @ param array $feed
* @ param string $uri
2020-07-28 17:37:36 +02:00
* @ param HttpSocket | null $HttpSocket Null can be for local feed
2019-08-26 08:53:35 +02:00
* @ return string
* @ throws Exception
*/
2021-10-03 17:03:24 +02:00
private function feedGetUri ( $feed , $uri , HttpSocket $HttpSocket = null )
2019-08-26 08:53:35 +02:00
{
2019-08-26 19:30:53 +02:00
if ( $this -> isFeedLocal ( $feed )) {
2019-08-26 08:53:35 +02:00
if ( file_exists ( $uri )) {
$data = file_get_contents ( $uri );
2019-08-25 12:23:33 +02:00
if ( $data === false ) {
2019-08-26 08:53:35 +02:00
throw new Exception ( " Could not read local file ' $uri '. " );
2019-08-25 12:23:33 +02:00
}
2020-04-28 15:31:27 +02:00
return $data ;
2019-08-25 12:23:33 +02:00
} else {
2019-08-26 08:53:35 +02:00
throw new Exception ( " Local file ' $uri ' doesn't exists. " );
2019-08-25 12:23:33 +02:00
}
2020-04-28 15:31:27 +02:00
}
2021-10-03 17:03:24 +02:00
return $this -> feedGetUriRemote ( $feed , $uri , $HttpSocket ) -> body ;
}
2020-07-28 17:37:36 +02:00
2021-10-03 17:03:24 +02:00
/**
* @ param array $feed
* @ param string $uri
* @ param HttpSocket $HttpSocket
* @ param string | null $etag
* @ return false | HttpSocketResponse
* @ throws HttpSocketHttpException
*/
private function feedGetUriRemote ( array $feed , $uri , HttpSocket $HttpSocket , $etag = null )
{
2020-04-28 15:31:27 +02:00
$request = $this -> __createFeedRequest ( $feed [ 'Feed' ][ 'headers' ]);
2021-10-03 17:03:24 +02:00
if ( $etag ) {
$request [ 'header' ][ 'If-None-Match' ] = $etag ;
}
2020-04-28 15:31:27 +02:00
2021-01-06 20:07:55 +01:00
try {
2021-10-03 17:03:24 +02:00
$response = $this -> getFollowRedirect ( $HttpSocket , $uri , $request );
2021-01-06 20:07:55 +01:00
} catch ( Exception $e ) {
throw new Exception ( " Fetching the ' $uri ' failed with exception: { $e -> getMessage () } " , 0 , $e );
2020-04-28 15:31:27 +02:00
}
2019-08-26 19:25:54 +02:00
2021-01-06 20:07:55 +01:00
if ( $response -> code != 200 ) { // intentionally !=
2021-10-03 17:03:24 +02:00
throw new HttpSocketHttpException ( $response , $uri );
2020-04-28 15:31:27 +02:00
}
$contentType = $response -> getHeader ( 'Content-Type' );
if ( $contentType === 'application/zip' ) {
$zipFile = new File ( $this -> tempFileName ());
2021-10-03 17:03:24 +02:00
$zipFile -> write ( $response -> body );
2020-04-28 15:31:27 +02:00
$zipFile -> close ();
2019-08-26 19:25:54 +02:00
2020-04-28 15:31:27 +02:00
try {
2021-10-03 17:03:24 +02:00
$response -> body = $this -> unzipFirstFile ( $zipFile );
2020-04-28 15:31:27 +02:00
} catch ( Exception $e ) {
throw new Exception ( " Fetching the ' $uri ' failed: { $e -> getMessage () } " );
} finally {
$zipFile -> delete ();
2019-08-25 12:23:33 +02:00
}
}
2021-10-03 17:03:24 +02:00
return $response ;
2019-08-25 12:23:33 +02:00
}
2019-08-26 19:30:53 +02:00
2019-08-28 18:20:28 +02:00
/**
* It should be possible to use 'redirect' $request attribute , but because HttpSocket contains bug that require
* certificate for first domain even when redirect to another domain , we need to use own solution .
*
* @ param HttpSocket $HttpSocket
* @ param string $url
* @ param array $request
* @ param int $iterations
* @ return false | HttpSocketResponse
* @ throws Exception
*/
private function getFollowRedirect ( HttpSocket $HttpSocket , $url , $request , $iterations = 5 )
{
for ( $i = 0 ; $i < $iterations ; $i ++ ) {
$response = $HttpSocket -> get ( $url , array (), $request );
if ( $response -> isRedirect ()) {
$HttpSocket = $this -> __setupHttpSocket ( null ); // Replace $HttpSocket with fresh instance
$url = trim ( $response -> getHeader ( 'Location' ), '=' );
} else {
return $response ;
}
}
throw new Exception ( " Maximum number of iteration reached. " );
}
2019-08-26 19:30:53 +02:00
/**
* @ param array $feed
* @ return bool
*/
private function isFeedLocal ( $feed )
{
return isset ( $feed [ 'Feed' ][ 'input_source' ]) && $feed [ 'Feed' ][ 'input_source' ] === 'local' ;
}
2019-08-26 19:47:25 +02:00
/**
* @ param int | null $jobId
* @ param string | null $message
* @ param int | null $progress
*/
private function jobProgress ( $jobId = null , $message = null , $progress = null )
{
if ( $jobId ) {
2020-09-28 18:39:43 +02:00
if ( ! isset ( $this -> Job )) {
$this -> Job = ClassRegistry :: init ( 'Job' );
}
$this -> Job -> saveProgress ( $jobId , $message , $progress );
2019-08-26 19:47:25 +02:00
}
}
2019-08-26 21:43:41 +02:00
2019-09-13 10:48:43 +02:00
/**
* remove all events tied to a feed . Returns int on success , error message
* as string on failure
*/
public function cleanupFeedEvents ( $user_id , $id )
{
$feed = $this -> find ( 'first' , array (
'conditions' => array ( 'Feed.id' => $id ),
'recursive' => - 1
));
if ( empty ( $feed )) {
return __ ( 'Invalid feed id.' );
}
if ( ! in_array ( $feed [ 'Feed' ][ 'source_format' ], array ( 'csv' , 'freetext' ))) {
return __ ( 'Feed has to be either a CSV or a freetext feed for the purging to work.' );
}
$this -> User = ClassRegistry :: init ( 'User' );
$user = $this -> User -> getAuthUser ( $user_id );
if ( empty ( $user )) {
return __ ( 'Invalid user id.' );
}
$conditions = array ( 'Event.info' => $feed [ 'Feed' ][ 'name' ] . ' feed' );
$this -> Event = ClassRegistry :: init ( 'Event' );
$events = $this -> Event -> find ( 'list' , array (
'conditions' => $conditions ,
'fields' => array ( 'Event.id' , 'Event.id' )
));
$count = count ( $events );
foreach ( $events as $event_id ) {
$this -> Event -> delete ( $event_id );
}
$this -> Log = ClassRegistry :: init ( 'Log' );
$this -> Log -> create ();
$this -> Log -> save ( array (
2021-06-24 12:07:09 +02:00
'org' => 'SYSTEM' ,
'model' => 'Feed' ,
'model_id' => $id ,
'email' => $user [ 'email' ],
'action' => 'purge_events' ,
'title' => __ ( 'Events related to feed %s purged.' , $id ),
'change' => null ,
2019-09-13 10:48:43 +02:00
));
$feed [ 'Feed' ][ 'fixed_event' ] = 1 ;
$feed [ 'Feed' ][ 'event_id' ] = 0 ;
$this -> save ( $feed );
return $count ;
}
2020-04-28 15:31:27 +02:00
/**
* @ param File $zipFile
* @ return string Uncompressed data
* @ throws Exception
*/
private function unzipFirstFile ( File $zipFile )
{
if ( ! class_exists ( 'ZipArchive' )) {
2020-10-07 09:07:11 +02:00
throw new Exception ( 'ZIP archive decompressing is not supported. ZIP extension is missing in PHP.' );
2020-04-28 15:31:27 +02:00
}
$zip = new ZipArchive ();
$result = $zip -> open ( $zipFile -> pwd ());
if ( $result !== true ) {
2020-10-07 09:07:11 +02:00
$errorCodes = [
ZipArchive :: ER_EXISTS => 'file already exists' ,
ZipArchive :: ER_INCONS => 'zip archive inconsistent' ,
ZipArchive :: ER_INVAL => 'invalid argument' ,
ZipArchive :: ER_MEMORY => 'malloc failure' ,
ZipArchive :: ER_NOENT => 'no such file' ,
ZipArchive :: ER_NOZIP => 'not a zip archive' ,
ZipArchive :: ER_OPEN => 'can\'t open file' ,
ZipArchive :: ER_READ => 'read error' ,
ZipArchive :: ER_SEEK => 'seek error' ,
];
$message = isset ( $errorCodes [ $result ]) ? $errorCodes [ $result ] : 'error ' . $result ;
throw new Exception ( " Remote server returns ZIP file, that cannot be open ( $message ) " );
2020-04-28 15:31:27 +02:00
}
if ( $zip -> numFiles !== 1 ) {
throw new Exception ( " Remote server returns ZIP file, that contains multiple files. " );
}
$filename = $zip -> getNameIndex ( 0 );
if ( $filename === false ) {
throw new Exception ( " Remote server returns ZIP file, but there is a problem with reading filename. " );
}
$zip -> close ();
$destinationFile = $this -> tempFileName ();
$result = copy ( " zip:// { $zipFile -> pwd () } # $filename " , $destinationFile );
if ( $result === false ) {
2020-10-07 09:07:11 +02:00
throw new Exception ( " Remote server returns ZIP file, that contains ' $filename ' file, but this file cannot be extracted. " );
2020-04-28 15:31:27 +02:00
}
$unzipped = new File ( $destinationFile );
$data = $unzipped -> read ();
if ( $data === false ) {
throw new Exception ( " Couldn't read extracted file content. " );
}
$unzipped -> delete ();
return $data ;
}
2016-02-29 22:32:04 +01:00
}