MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: Local feeds 

- still needs testing
pull/1881/head
iglocska 2017-01-24 14:07:55 +01:00
parent 8c63e6f3d5
commit aba2491344
3 changed files with 156 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Controller/FeedsController.php
View File

@ -150,6 +150,9 @@ class FeedsController extends AppController {
if (empty($this->request->data['Feed']['target_event'])) {
$this->request->data['Feed']['target_event'] = 0;
}
if (empty($this->request->data['Feed']['input_source'])) {
$this->request->data['Feed']['input_source'] = 'network';
}
$this->request->data['Feed']['settings'] = json_encode($this->request->data['Feed']['settings']);
$this->request->data['Feed']['event_id'] = !empty($this->request->data['Feed']['fixed_event']) ? $this->request->data['Feed']['target_event'] : 0;
if (!$error) {
@ -212,7 +215,7 @@ class FeedsController extends AppController {
$this->request->data['Feed']['settings']['delimiter'] = ',';
}
$this->request->data['Feed']['settings'] = json_encode($this->request->data['Feed']['settings']);
$fields = array('id', 'name', 'provider', 'enabled', 'rules', 'url', 'distribution', 'sharing_group_id', 'tag_id', 'fixed_event', 'event_id', 'publish', 'delta_merge', 'override_ids', 'settings');
$fields = array('id', 'name', 'provider', 'enabled', 'rules', 'url', 'distribution', 'sharing_group_id', 'tag_id', 'fixed_event', 'event_id', 'publish', 'delta_merge', 'override_ids', 'settings', 'input_source');
$feed = array();
foreach ($fields as $field) {
if (isset($this->request->data['Feed'][$field])) {

36
app/Model/AppModel.php
View File

@ -40,7 +40,7 @@ class AppModel extends Model {
32 => false, 33 => true, 38 => true, 39 => true, 40 => false,
42 => false, 44 => false, 45 => false, 49 => true, 50 => false,
51 => false, 52 => false, 55 => true, 56 => true, 57 => true,
58 => false, 59 => false, 60 => false
58 => false, 59 => false, 60 => false, 61 => false
)
)
);
@ -113,6 +113,8 @@ class AppModel extends Model {
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
$sql = '';
$sqlArray = array();
$indexArray = array();
$this->Log = ClassRegistry::init('Log');
$clean = true;
switch ($command) {
@ -558,6 +560,10 @@ class AppModel extends Model {
$this->__dropIndex('attribute_tags', 'attribute_id');
$this->__dropIndex('attribute_tags', 'tag_id');
break;
case '2.4.61':
$sqlArray[] = 'ALTER TABLE feeds ADD input_source varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "network";';
$indexArray[] = array('feeds', 'input_source');
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -606,6 +612,34 @@ class AppModel extends Model {
'change' => 'The executed SQL query was: ' . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage()
));
}
foreach ($indexArray as $iA) {
try {
$this->__addIndex($iA[0], $iA[1]);
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Successfuly executed the SQL query for ' . $command,
'change' => 'New index for field ' . $iA[1] . ' added to table ' . $iA[0],
));
} catch (Exception $e) {
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Issues executing the SQL query for ' . $command,
'change' => 'Failed to add index for field ' . $iA[1] . ' for table ' . $iA[0],
));
}
}
}
if ($clean) $this->cleanCacheFiles();
return true;

166
app/Model/Feed.php
View File

@ -23,8 +23,8 @@ class Feed extends AppModel {
public $validate = array(
'url' => array( // TODO add extra validation to refuse multiple time the same url from the same org
'rule' => array('url'),
'message' => 'Please enter a valid url.',
'rule' => array('urlOrExistingFilepath'),
'message' => 'Please enter a valid url or file path (make sure that the choice matches the input source setting).',
),
'provider' => 'valueNotEmpty',
'name' => 'valueNotEmpty',
@ -47,6 +47,20 @@ class Feed extends AppModel {
)
);
public function urlOrExistingFilepath($fields) {
$input_source = empty($this->data['Feed']['input_source']) ? 'network' : $this->data['Feed']['input_source'];
if ($input_source == 'local') {
if (!file_exists($this->data['Feed']['url']) && !is_dir($this->data['Feed']['url'])) {
return false;
}
} else {
if (!filter_var($url, FILTER_VALIDATE_URL)) {
return false;
}
}
return true;
}
public function getFeedTypesOptions() {
$result = array();
foreach ($this->feed_types as $key => $value) {
@ -59,11 +73,19 @@ class Feed extends AppModel {
// returns an array with the UUIDs of events that are new or that need updating
public function getNewEventUuids($feed, $HttpSocket) {
$result = array();
$request = $this->__createFeedRequest();
$uri = $feed['Feed']['url'] . '/manifest.json';
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != 200) return 1;
$manifest = json_decode($response->body, true);
if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
if (file_exists($feed['Feed']['url'] . '/manifest.json')) {
$data = file_get_contents($feed['Feed']['url'] . '/manifest.json');
}
} else {
$request = $this->__createFeedRequest();
$uri = $feed['Feed']['url'] . '/manifest.json';
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != 200) return 1;
$data = $response->body;
unset($response);
}
$manifest = json_decode($data, true);
if (!$manifest) return 2;
$this->Event = ClassRegistry::init('Event');
$events = $this->Event->find('all', array(
@ -85,10 +107,21 @@ class Feed extends AppModel {
public function getManifest($feed, $HttpSocket) {
$result = array();
$request = $this->__createFeedRequest();
$uri = $feed['Feed']['url'] . '/manifest.json';
$response = $HttpSocket->get($uri, '', $request);
if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
if (file_exists($feed['Feed']['url'] . '/manifest.json')) {
$data = file_get_contents($feed['Feed']['url'] . '/manifest.json');
}
} else {
$uri = $feed['Feed']['url'] . '/manifest.json';
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != 200) {
return false;
}
$data = $response->body;
unset($response);
}
try {
$events = json_decode($response->body, true);
$events = json_decode($data, true);
} catch (Exception $e) {
return false;
}
@ -98,20 +131,26 @@ class Feed extends AppModel {
public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) {
$result = array();
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . intval($feed['Feed']['id']) . '.cache';
$doFetch = true;
if (file_exists($feedCache)) {
$file = new File($feedCache);
if (time() - $file->lastChange() < 600) {
$doFetch = false;
$data = file_get_contents($feedCache);
if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
if (file_exists($feed['Feed']['url'])) {
$data = file_get_contents($feed['Feed']['url']);
}
}
if ($doFetch) {
$response = $HttpSocket->get($feed['Feed']['url'], '', array());
if ($response->code == 200) {
$data = $response->body;
file_put_contents($feedCache, $data);
} else {
$feedCache = APP . 'tmp' . DS . 'cache' . DS . 'misp_feed_' . intval($feed['Feed']['id']) . '.cache';
$doFetch = true;
if (file_exists($feedCache)) {
$file = new File($feedCache);
if (time() - $file->lastChange() < 600) {
$doFetch = false;
$data = file_get_contents($feedCache);
}
}
if ($doFetch) {
$response = $HttpSocket->get($feed['Feed']['url'], '', array());
if ($response->code == 200) {
$data = $response->body;
file_put_contents($feedCache, $data);
}
}
}
App::uses('ComplexTypeTool', 'Tools');
@ -309,15 +348,22 @@ class Feed extends AppModel {
}
public function downloadEventFromFeed($feed, $uuid, $user) {
$HttpSocket = $this->__setupHttpSocket($feed);
$request = $this->__createFeedRequest();
$uri = $feed['Feed']['url'] . '/' . $uuid . '.json';
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != 200) {
return false;
$path = $feed['Feed']['url'] . '/' . $uuid . '.json';
if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
if (file_exists($path)) {
$data = file_get_contents($path);
}
} else {
return $this->__prepareEvent($response->body, $feed);
$HttpSocket = $this->__setupHttpSocket($feed);
$request = $this->__createFeedRequest();
$response = $HttpSocket->get($path, '', $request);
if ($response->code != 200) {
return false;
}
$data = $response->body;
unset($response->body);
}
return $this->__prepareEvent($data, $feed);
}
private function __saveEvent($event, $user) {
@ -392,31 +438,49 @@ class Feed extends AppModel {
}
private function __addEventFromFeed($HttpSocket, $feed, $uuid, $user, $filterRules) {
$request = $this->__createFeedRequest();
$uri = $feed['Feed']['url'] . '/' . $uuid . '.json';
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != 200) {
if (!Validation::uuid($uuid)) {
return false;
} else {
$event = $this->__prepareEvent($response->body, $feed);
if (is_array($event)) {
$this->Event = ClassRegistry::init('Event');
return $this->Event->_add($event, true, $user);
} else return $event;
}
$path = $feed['Feed']['url'] . '/' . $uuid . '.json';
if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
if (file_exists($path)) {
$data = file_get_contents($path);
}
} else {
$request = $this->__createFeedRequest();
$response = $HttpSocket->get($path, '', $request);
if ($response->code != 200) {
return false;
}
$data = $response->body;
unset($response);
}
$event = $this->__prepareEvent($data, $feed);
if (is_array($event)) {
$this->Event = ClassRegistry::init('Event');
return $this->Event->_add($event, true, $user);
} else return $event;
}
private function __updateEventFromFeed($HttpSocket, $feed, $uuid, $eventId, $user, $filterRules) {
$request = $this->__createFeedRequest();
$uri = $feed['Feed']['url'] . '/' . $uuid . '.json';
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != 200) {
if (!Validation::uuid($uuid)) {
return false;
} else {
$event = $this->__prepareEvent($response->body, $feed);
$this->Event = ClassRegistry::init('Event');
return $this->Event->_edit($event, $user, $uuid, $jobId = null);
}
$path = $feed['Feed']['url'] . '/' . $uuid . '.json';
if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
if (file_exists($path)) {
$data = file_get_contents($path);
}
} else {
$request = $this->__createFeedRequest();
$response = $HttpSocket->get($path, '', $request);
if ($response->code != 200) {
return false;
}
}
$event = $this->__prepareEvent($response->body, $feed);
$this->Event = ClassRegistry::init('Event');
return $this->Event->_edit($event, $user, $uuid, $jobId = null);
}
public function addDefaultFeeds($newFeeds) {
@ -450,7 +514,11 @@ class Feed extends AppModel {
if (isset($this->data['Feed']['settings']) && !empty($this->data['Feed']['settings'])) {
$this->data['Feed']['settings'] = json_decode($this->data['Feed']['settings'], true);
}
$HttpSocket = $syncTool->setupHttpSocketFeed($this->data);
if (isset($this->data['Feed']['input_source']) && $this->data['Feed']['input_source'] == 'local') {
$HttpSocket = false;
} else {
$HttpSocket = $syncTool->setupHttpSocketFeed($this->data);
}
if ($this->data['Feed']['source_format'] == 'misp') {
if ($jobId) {
$job->id = $jobId;