MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into stix2

pull/3707/head
chrisr3d 2018-08-03 13:51:33 +02:00
parent afbb83a827 aba4c90e0c
commit 0ed3f0617c
7 changed files with 336 additions and 519 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Controller/EventsController.php
View File

@ -2433,7 +2433,7 @@ class EventsController extends AppController
// For XML: <request><value>7.7.7.7&amp;&amp;1.1.1.1</value><type>ip-src</type></request>
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request".'));
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").'));
} else {
$data = $this->request->data;
}
@ -4064,7 +4064,7 @@ class EventsController extends AppController
// This would return all OSINT tagged events except for event #3 and #4
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request".'));
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").'));
} else {
$data = $this->request->data;
}

55
app/Model/AppModel.php
View File

@ -1461,12 +1461,12 @@ class AppModel extends Model
}
// generate a generic subquery - options needs to include conditions
public function subQueryGenerator($model, $options, $lookupKey)
public function subQueryGenerator($model, $options, $lookupKey, $negation = false)
{
$db = $model->getDataSource();
$defaults = array(
'fields' => array('*'),
'table' => $model->alias,
'table' => $model->table,
'alias' => $model->alias,
'limit' => null,
'offset' => null,
@ -1476,17 +1476,21 @@ class AppModel extends Model
);
$params = array();
foreach (array_keys($defaults) as $key) {
if (isset($conditions[$key])) {
$params[$key] = $conditions[$key];
if (isset($options[$key])) {
$params[$key] = $options[$key];
} else {
$params[$key] = $conditions[$key];
$params[$key] = $defaults[$key];
}
}
$subQuery = $db->buildStatement(
$params,
$model
);
$subQuery = $lookupKey . ' IN (' . $subQuery . ') ';
if ($negation) {
$subQuery = $lookupKey . ' NOT IN (' . $subQuery . ') ';
} else {
$subQuery = $lookupKey . ' IN (' . $subQuery . ') ';
}
$conditions = array(
$db->expression($subQuery)->value
);
@ -1581,4 +1585,43 @@ class AppModel extends Model
}
return true;
}
public function setupHttpSocket($server, $HttpSocket = null)
{
if (empty($HttpSocket)) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
return $HttpSocket;
}
public function setupSyncRequest($server)
{
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json'
)
);
$request = $this->addHeaders($request);
return $request;
}
public function addHeaders($request)
{
$version = $this->checkMISPVersion();
$version = implode('.', $version);
try {
$commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
} catch (Exception $e) {
$commit = false;
}
$request['header']['MISP-version'] = $version;
if ($commit) {
$request['header']['commit'] = $commit;
}
return $request;
}
}

92
app/Model/Attribute.php
View File

@ -299,7 +299,7 @@ class Attribute extends AppModel
),
'Network activity' => array(
'desc' => 'Information about network traffic generated by the malware',
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-sha1', 'other', 'hex', 'cookie')
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-sha1', 'other', 'hex', 'cookie', 'hostname|port')
),
'Payload type' => array(
'desc' => 'Information about the final payload(s)',
@ -2043,17 +2043,53 @@ class Attribute extends AppModel
// If we sent any tags along, load the associated tag names for each attribute
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$tagArray = $tag->fetchTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
if (!empty($tagArray[0])) {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id'));
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id'));
$temp2 = array('OR' => $temp);
$conditions['AND'][] = $temp2;
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
if (!empty($tagArray[1])) {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id', 1));
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id', 1));
$temp2 = array('AND' => $temp);
$conditions['AND'][] = $temp2;
}
$conditions['AND'][] = $temp;
}
$attributes = $this->fetchAttributes($user, array(
'conditions' => $conditions,
@ -2524,28 +2560,11 @@ class Attribute extends AppModel
{
$conditions = array();
if (!$user['Role']['perm_site_admin']) {
$sgids = $this->SharingGroup->fetchAllAuthorised($user);
$sgids = $this->Event->cacheSgids($user, true);
$eventConditions = $this->Event->createEventConditions($user);
$conditions = array(
'AND' => array(
array(
'OR' => array(
'Event.org_id' => $user['org_id'],
array(
'AND' => array(
'Event.distribution >' => 0,
'Event.distribution <' => 4,
Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
),
),
array(
'AND' => array(
'Event.sharing_group_id' => $sgids,
'Event.distribution' => 4,
Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
)
)
)
),
$eventConditions['AND'],
array(
'OR' => array(
'Event.org_id' => $user['org_id'],
@ -2751,20 +2770,25 @@ class Attribute extends AppModel
}
if (empty($params['limit'])) {
$pagesToFetch = $this->find('count', array('conditions' => $params['conditions']));
$loopLimit = 100000;
$pagesToFetch = ceil($pagesToFetch / $loopLimit);
$loop = true;
$params['limit'] = $loopLimit;
$params['page'] = 0;
} else {
$loop = false;
$pagesToFetch = 1;
}
$attributes = array();
for ($i = 0; $i < $pagesToFetch; $i++) {
$continue = true;
while ($continue) {
if ($loop) {
$params['limit'] = $loopLimit;
$params['page'] = $i+1;
$params['page'] = $params['page'] + 1;
if (isset($results) && count($results) < $loopLimit) {
$continue = false;
continue;
}
} else {
$continue = false;
}
$results = $this->find('all', $params);
// return false if we're paginating

383
app/Model/Event.php
View File

@ -257,13 +257,7 @@ class Event extends AppModel
)
);
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
}
// The Associations below have been created with all possible keys, those that are not needed can be removed
public $belongsTo = array(
'User' => array(
'className' => 'User',
@ -853,64 +847,19 @@ class Event extends AppModel
switch ($code) {
case 403:
return 'The distribution level of this event blocks it from being pushed.';
break;
case 405:
$error = 'The sync user on the remote instance does not have the required privileges to handle this event.';
break;
}
if ($error) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => 'SYSTEM',
'action' => 'warning',
'user_id' => 0,
'title' => 'Uploading Event (' . $event['Event']['id'] . ') to Server (' . $server['Server']['id'] . ')',
'change' => 'Remote instance returned an error, with error code: ' . $code,
));
$newTextBody = 'Uploading Event (' . $event['Event']['id'] . ') to Server (' . $server['Server']['id'] . ')';
$this->__logUploadResult($server, $event, $newTextBody);
}
return $error;
}
public function uploadEventToServer($event, $server, $HttpSocket = null)
private function __executeRestfulEventToServer($event, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket)
{
$this->Server = ClassRegistry::init('Server');
$push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket);
if (!isset($push['canPush'])) {
$test = $this->Server->checkLegacyServerSyncPrivilege($server['Server']['id'], $HttpSocket);
} else {
if (!$push['canPush']) {
return 'The remote user is not a sync user - the upload of the event has been blocked.';
}
}
$deletedAttributes = false;
if (($push['version'][0] > 2) ||
($push['version'][0] == 2 && $push['version'][1] > 4) ||
($push['version'][0] == 2 && $push['version'][1] == 4 && $push['version'][2] > 42)) {
$deletedAttributes = true;
}
if (isset($event['Attribute']) && !$deletedAttributes) {
foreach ($event['Attribute'] as $k => $v) {
if ($v['deleted']) {
unset($event['Attribute'][$k]);
}
}
$event['Attribute'] = array_values($event['Attribute']);
}
if (!isset($push['canPush']) || !$push['canPush']) {
return 'Trying to push to an outdated instance.';
}
if (isset($server['Server']['unpublish_event'])) {
$unpublish_event = $server['Server']['unpublish_event'];
if ($unpublish_event) {
$event['Event']['published'] = 0;
}
}
$updated = null;
$newLocation = $newTextBody = '';
$result = $this->restfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $event, $server);
@ -918,84 +867,47 @@ class Event extends AppModel
return $error . ' Error code: ' . $result;
}
}
if (strlen($newLocation) || $result) { // HTTP/1.1 200 OK or 302 Found and Location: http://<newLocation>
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
$result = $this->restfulEventToServer($event, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $event, $server);
if ($error) {
return $error . ' Error code: ' . $result;
}
}
}
$uploadFailed = false;
try {
$json = json_decode($newTextBody, true);
} catch (Exception $e) {
$uploadFailed = true;
}
if (!is_array($json) || $uploadFailed) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => 'SYSTEM',
'action' => 'warning',
'user_id' => 0,
'title' => 'Uploading Event (' . $event['Event']['id'] . ') to Server (' . $server['Server']['id'] . ')',
'change' => 'Returned message: ', $newTextBody,
));
return false;
}
// get the remote event_id
foreach ($json as $jsonEvent) {
if (is_array($jsonEvent)) {
foreach ($jsonEvent as $key => $value) {
if ($key == 'id') {
break;
}
}
} else {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => 'SYSTEM',
'action' => 'warning',
'user_id' => 0,
'title' => 'Uploading Event (' . $event['Event']['id'] . ') to Server (' . $server['Server']['id'] . ')',
'change' => 'Returned message: ', $newTextBody,
));
return false;
return true;
}
public function uploadEventToServer($event, $server, $HttpSocket = null)
{
$this->Server = ClassRegistry::init('Server');
$push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket);
if (empty($push['canPush'])) {
return 'The remote user is not a sync user - the upload of the event has been blocked.';
}
if (!empty($server['Server']['unpublish_event'])) {
$event['Event']['published'] = 0;
}
$updated = null;
$newLocation = $newTextBody = '';
$result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket);
if ($result !== true) {
return $result;
}
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
$result = $this->restfulEventToServer($event, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $event, $server);
if ($error) {
return $error . ' Error code: ' . $result;
}
}
}
$uploadFailed = false;
try {
$json = json_decode($newTextBody, true);
} catch (Exception $e) {
$uploadFailed = true;
}
if (!is_array($json) || $uploadFailed) {
return $this->__logUploadResult($server, $event, $newTextBody);
}
return 'Success';
}
public function addHeaders($request)
{
$version = $this->checkMISPVersion();
$version = implode('.', $version);
try {
$commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
} catch (Exception $e) {
$commit = false;
}
$request['header']['MISP-version'] = $version;
if ($commit) {
$request['header']['commit'] = $commit;
}
return $request;
}
// Uploads the event and the associated Attributes to another Server
public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
{
private function __prepareForPushToServer($event, $server) {
if ($event['Event']['distribution'] == 4) {
if (!empty($event['SharingGroup']['SharingGroupServer'])) {
$found = false;
@ -1015,36 +927,25 @@ class Event extends AppModel
return 403;
}
$server = $server[0];
if ($this->checkDistributionForPush($event, $server, $context = 'Event')) {
if ($this->checkDistributionForPush($event, $server, 'Event')) {
$event = $this->__updateEventForSync($event, $server);
} else {
return 403;
}
$url = $server['Server']['url'];
$authkey = $server['Server']['authkey'];
if (null == $HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$request = array(
'header' => array(
'Authorization' => $authkey,
'Accept' => 'application/json',
'Content-Type' => 'application/json',
return $event;
}
//'Connection' => 'keep-alive' // // LATER followup cakephp issue about this problem: https://github.com/cakephp/cakephp/issues/1961
)
);
$request = $this->addHeaders($request);
$uri = $url . '/events';
if (isset($urlPath)) {
private function __getLastUrlPathComponent($urlPath)
{
if (!empty($urlPath)) {
$pieces = explode('/', $urlPath);
$uri .= '/' . end($pieces);
return '/' . end($pieces);
}
$data = json_encode($event);
// LATER validate HTTPS SSL certificate
$response = $HttpSocket->post($uri, $data, $request);
return '';
}
private function __handleRestfulEventToServerResponse($response, &$newLocation, &$newTextBody)
{
switch ($response->code) {
case '200': // 200 (OK) + entity-action-result
if ($response->isOk()) {
@ -1055,34 +956,39 @@ class Event extends AppModel
try {
$jsonArray = json_decode($response->body, true);
} catch (Exception $e) {
return true; // TODO should be false
}
if (strpos($jsonArray['name'], "Event already exists")) { // strpos, so i can piggyback some value if needed.
return true;
} else {
return $jsonArray['name'];
}
return $jsonArray['name'];
}
break;
case '302': // Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return true;
break;
case '404': // Not Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return 404;
break;
case '405':
return 405;
break;
case '403': // Not authorised
return 403;
break;
}
}
// Uploads the event and the associated Attributes to another Server
public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
{
$event = $this->__prepareForPushToServer($event, $server);
if (is_numeric($event)) return $event;
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/events' . $this->__getLastUrlPathComponent($urlPath);
$data = json_encode($event);
$response = $HttpSocket->post($uri, $data, $request);
return $this->__handleRestfulEventToServerResponse($response, $newLocation, $newTextBody);
}
private function __updateEventForSync($event, $server)
{
// rearrange things to be compatible with the Xml::fromArray()
@ -1238,24 +1144,8 @@ class Event extends AppModel
public function downloadEventFromServer($eventId, $server, $HttpSocket=null)
{
$url = $server['Server']['url'];
$authkey = $server['Server']['authkey'];
if (null == $HttpSocket) {
//$HttpSocket = new HttpSocket(array(
// 'ssl_verify_peer' => false
// ));
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$request = array(
'header' => array(
'Authorization' => $authkey,
'Accept' => 'application/json',
'Content-Type' => 'application/json',
//'Connection' => 'keep-alive' // // LATER followup cakephp issue about this problem: https://github.com/cakephp/cakephp/issues/1961
)
);
$request = $this->addHeaders($request);
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/events/view/' . $eventId . '/deleted:1/excludeGalaxy:1';
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
@ -1354,21 +1244,8 @@ class Event extends AppModel
public function downloadProposalsFromServer($uuidList, $server, $HttpSocket = null)
{
$url = $server['Server']['url'];
$authkey = $server['Server']['authkey'];
if (null == $HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$request = array(
'header' => array(
'Authorization' => $authkey,
'Accept' => 'application/json',
'Content-Type' => 'application/json',
//'Connection' => 'keep-alive' // LATER followup cakephp issue about this problem: https://github.com/cakephp/cakephp/issues/1961
)
);
$request = $this->addHeaders($request);
$HttpSocket = $this->__setupHttpSocket($server, $HttpSocket);
$request = $this->__setupPushRequest($server);
$uri = $url . '/shadow_attributes/getProposalsByUuidList';
$response = $HttpSocket->post($uri, json_encode($uuidList), $request);
if ($response->isOk()) {
@ -1378,7 +1255,7 @@ class Event extends AppModel
}
}
private function __createEventConditions($user)
public function createEventConditions($user)
{
$conditions = array();
if (!$user['Role']['perm_site_admin']) {
@ -1406,7 +1283,7 @@ class Event extends AppModel
public function fetchSimpleEventIds($user, $params = array())
{
$conditions = $this->__createEventConditions($user);
$conditions = $this->createEventConditions($user);
$conditions['AND'][] = $params['conditions'];
$results = array_values($this->find('list', array(
'conditions' => $conditions,
@ -1418,7 +1295,7 @@ class Event extends AppModel
public function fetchSimpleEvents($user, $params, $includeOrgc = false)
{
$conditions = $this->__createEventConditions($user);
$conditions = $this->createEventConditions($user);
$conditions['AND'][] = $params['conditions'];
$params = array(
'conditions' => $conditions,
@ -1433,31 +1310,8 @@ class Event extends AppModel
public function fetchEventIds($user, $from = false, $to = false, $last = false, $list = false, $timestamp = false, $publish_timestamp = false, $eventIdList = false)
{
$conditions = array();
// restricting to non-private or same org if the user is not a site-admin.
if (!$user['Role']['perm_site_admin']) {
$sgids = $this->SharingGroup->fetchAllAuthorised($user);
if (empty($sgids)) {
$sgids = -1;
}
$conditions['AND']['OR'] = array(
'Event.org_id' => $user['org_id'],
array(
'AND' => array(
'Event.distribution >' => 0,
'Event.distribution <' => 4,
Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
),
),
array(
'AND' => array(
'Event.sharing_group_id' => $sgids,
'Event.distribution' => 4,
Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
)
)
);
}
$conditions = $this->createEventConditions($user);
$fields = array('Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id');
if ($from) {
@ -1562,10 +1416,9 @@ class Event extends AppModel
$options[$opt] = false;
}
}
$conditions = $this->createEventConditions($user);
if ($options['eventid']) {
$conditions['AND'][] = array("Event.id" => $options['eventid']);
} else {
$conditions = array();
}
if ($options['eventsExtendingUuid']) {
if (!is_array($options['eventsExtendingUuid'])) {
@ -1613,23 +1466,6 @@ class Event extends AppModel
$sgids = $this->cacheSgids($user, $useCache);
// restricting to non-private or same org if the user is not a site-admin.
if (!$isSiteAdmin) {
$conditions['AND']['OR'] = array(
'Event.org_id' => $user['org_id'],
array(
'AND' => array(
'Event.distribution >' => 0,
'Event.distribution <' => 4,
Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
),
),
array(
'AND' => array(
'Event.sharing_group_id' => $sgids,
'Event.distribution' => 4,
Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array()
)
)
);
// if delegations are enabled, check if there is an event that the current user might see because of the request itself
if (Configure::read('MISP.delegation')) {
$delegatedEventIDs = $this->__cachedelegatedEventIDs($user, $useCache);
@ -3356,21 +3192,25 @@ class Event extends AppModel
}
}
private function __getPrioWorkerIfPossible() {
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$workerType = 'default';
foreach ($workers as $worker) {
if ($worker['queue'] === 'prio') {
$workerType = 'prio';
}
}
return $workerType;
}
public function publishRouter($id, $passAlong = null, $user)
{
if (Configure::read('MISP.background_jobs')) {
$job = ClassRegistry::init('Job');
$job->create();
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$workerType = 'default';
foreach ($workers as $worker) {
if ($worker['queue'] === 'prio') {
$workerType = 'prio';
}
}
$data = array(
'worker' => $workerType,
'worker' => $this->__getPrioWorkerIfPossible(),
'job_type' => 'publish_event',
'job_input' => 'Event ID: ' . $id,
'status' => 0,
@ -3514,9 +3354,7 @@ class Event extends AppModel
$i = 0;
foreach ($events as $k => $event) {
$this->set($event);
if ($this->validates()) {
// validates
} else {
if (!$this->validates()) {
$errors = $this->validationErrors;
$result[$i]['id'] = $event['Event']['id'];
$result[$i]['error'] = $errors;
@ -3670,6 +3508,7 @@ class Event extends AppModel
$event_ids = array_intersect($event_ids, $idList);
}
$randomFileName = $this->generateRandomFileName();
<<<<<<< HEAD
$tmpDir = APP . "files" . DS . "scripts";
$stix2_framing_cmd = 'python3 ' . $tmpDir . DS . 'misp_framing.py stix2 ' . escapeshellarg(CakeText::uuid()) . ' 2>' . APP . 'tmp/logs/exec-errors.log';
$stix2_framing = json_decode(shell_exec($stix2_framing_cmd), true);
@ -3686,6 +3525,23 @@ class Event extends AppModel
if (!$this->Job->exists()) {
$jobId = false;
}
=======
$tmpDir = APP . "files" . DS . "scripts" . DS . "tmp";
$tempFile = new File($tmpDir . DS . $randomFileName, true, 0644);
$tempFile->write($event);
$scriptFile = APP . "files" . DS . "scripts" . DS . "stix2" . DS . "misp2stix2.py";
$result = shell_exec('python3 ' . $scriptFile . ' ' . $tempFile->path . ' json ' . escapeshellarg(Configure::read('MISP.baseurl')) . ' ' . escapeshellarg(Configure::read('MISP.org')) . ' 2>' . APP . 'tmp/logs/exec-errors.log');
$tempFile->delete();
$resultFile = new File($tmpDir . DS . $randomFileName . ".stix2");
$resultFile->write("{\"type\": \"bundle\", \"spec_version\": \"2.0\", \"id\": \"bundle--" . CakeText::uuid() . "\", \"objects\": [");
if (trim($result) == 1) {
$file = new File($tmpDir . DS . $randomFileName . '.out', true, 0644);
$result = substr($file->read(), 1, -1);
$file->delete();
$resultFile->append($result);
} else {
return false;
>>>>>>> aba4c90e0c9627ef1e70e6da71dc2f3c5c7dd8c5
}
$i = 0;
$eventCount = count($event_ids);
@ -4770,16 +4626,8 @@ class Event extends AppModel
if (Configure::read('MISP.background_jobs')) {
$job = ClassRegistry::init('Job');
$job->create();
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$workerType = 'default';
foreach ($workers as $worker) {
if ($worker['queue'] === 'prio') {
$workerType = 'prio';
}
}
$data = array(
'worker' => $workerType,
'worker' => $this->__getPrioWorkerIfPossible(),
'job_type' => 'enrichment',
'job_input' => 'Event ID: ' . $options['event_id'] . ' modules: ' . json_encode($options['modules']),
'status' => 0,
@ -4917,4 +4765,21 @@ class Event extends AppModel
$eventLock = ClassRegistry::init('EventLock');
$eventLock->insertLock($user, $id);
}
private function __logUploadResult($server, $event, $newTextBody)
{
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => 'SYSTEM',
'action' => 'warning',
'user_id' => 0,
'title' => 'Uploading Event (' . $event['Event']['id'] . ') to Server (' . $server['Server']['id'] . ')',
'change' => 'Returned message: ', $newTextBody,
));
return false;
}
}

286
app/Model/Server.php
View File

@ -5,7 +5,7 @@ class Server extends AppModel
{
public $name = 'Server';
public $actsAs = array('SysLogLogable.SysLogLogable' => array( // TODO Audit, logable, check: 'userModel' and 'userKey' can be removed given default
public $actsAs = array('SysLogLogable.SysLogLogable' => array(
'userModel' => 'User',
'userKey' => 'user_id',
'change' => 'full'
@ -40,7 +40,7 @@ class Server extends AppModel
public $displayField = 'url';
public $validate = array(
'url' => array( // TODO add extra validation to refuse multiple time the same url from the same org
'url' => array(
'url' => array(
'rule' => array('url'),
'message' => 'Please enter a valid base-url.'
@ -134,7 +134,6 @@ class Server extends AppModel
)
);
// TODO i18n
$this->serverSettings = array(
'MISP' => array(
'branch' => 1,
@ -1781,9 +1780,6 @@ class Server extends AppModel
'recursive' => -1,
));
$eventIds = array_intersect($eventIds, $local_event_ids);
} elseif ("incremental" === $technique) {
// TODO incremental pull
return array(3, null);
} elseif (is_numeric($technique)) {
$eventIds[] = intval($technique);
// if we are downloading a single event, don't fetch all proposals
@ -1798,9 +1794,7 @@ class Server extends AppModel
if (!empty($eventIds)) {
// download each event
if (null != $eventIds) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$HttpSocket = $this->setupHttpSocket($server);
foreach ($eventIds as $k => $eventId) {
$event = $eventModel->downloadEventFromServer(
$eventId,
@ -2008,25 +2002,13 @@ class Server extends AppModel
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false)
{
$url = $server['Server']['url'];
$authkey = $server['Server']['authkey'];
if ($ignoreFilterRules) {
$filter_rules = array();
} else {
$filter_rules = $this->filterRuleToParameter($server['Server']['pull_rules']);
}
if (null == $HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$request = array(
'header' => array(
'Authorization' => $authkey,
'Accept' => 'application/json',
'Content-Type' => 'application/json',
//'Connection' => 'keep-alive' // LATER followup cakephp issue about this problem: https://github.com/cakephp/cakephp/issues/1961
)
);
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/events/index';
$filter_rules['minimal'] = 1;
try {
@ -2086,7 +2068,6 @@ class Server extends AppModel
return 403;
}
} catch (SocketException $e) {
// FIXME refactor this with clean try catch over all http functions
return $e->getMessage();
}
// error, so return error message, since that is handled and everything is expecting an array
@ -2259,19 +2240,9 @@ class Server extends AppModel
}
unset($eventIds[$k]['Event']['id']);
}
if (null == $HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$data = json_encode($eventIds);
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
$uri = $server['Server']['url'] . '/events/filterEventIdsForPush';
$response = $HttpSocket->post($uri, $data, $request);
if ($response->code == '200') {
@ -2285,11 +2256,7 @@ class Server extends AppModel
public function syncProposals($HttpSocket, $server, $sa_id = null, $event_id = null, $eventModel)
{
$saModel = ClassRegistry::init('ShadowAttribute');
if (null == $HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
if ($sa_id == null) {
if ($event_id == null) {
// event_id is null when we are doing a push
@ -2323,13 +2290,7 @@ class Server extends AppModel
}
$data = json_encode($event['ShadowAttribute']);
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/events/pushProposals/' . $event['Event']['uuid'];
$response = $HttpSocket->post($uri, $data, $request);
if ($response->code == '200') {
@ -2351,13 +2312,7 @@ class Server extends AppModel
}
} else {
// connect to checkuuid($uuid)
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/events/checkuuid/' . $sa_id;
$response = $HttpSocket->get($uri, '', $request);
if ($response->code != '200') {
@ -2372,6 +2327,13 @@ class Server extends AppModel
$this->Module = ClassRegistry::init('Module');
$serverSettings = $this->serverSettings;
$moduleTypes = array('Enrichment', 'Import', 'Export', 'Cortex');
$serverSettings = $this->readModuleSettings($serverSettings, $moduleTypes);
return $serverSettings;
}
private function readModuleSettings($serverSettings, $moduleTypes)
{
$this->Module = ClassRegistry::init('Module');
$orgs = $this->Organisation->find('list', array(
'conditions' => array(
'Organisation.local' => 1
@ -2390,7 +2352,6 @@ class Server extends AppModel
if ($result['type'] == 'boolean') {
$setting['test'] = 'testBool';
$setting['type'] = 'boolean';
$setting['description'] = __('Enable or disable the %s module.', $module);
$setting['value'] = false;
} elseif ($result['type'] == 'orgs') {
@ -2413,37 +2374,13 @@ class Server extends AppModel
return $serverSettings;
}
# TODO [i18n] think about it
public function serverSettingsRead($unsorted = false)
{
$this->Module = ClassRegistry::init('Module');
$serverSettings = $this->getCurrentServerSettings();
$currentSettings = Configure::read();
if (Configure::read('Plugin.Enrichment_services_enable')) {
$results = $this->Module->getModuleSettings();
foreach ($results as $module => $data) {
foreach ($data as $result) {
$setting = array('level' => 1, 'errorMessage' => '');
if ($result['type'] == 'boolean') {
$setting['test'] = 'testBool';
$setting['type'] = 'boolean';
$setting['description'] = __('Enable or disable the %s module.', $module);
$setting['value'] = false;
} elseif ($result['type'] == 'orgs') {
$setting['description'] = __('Restrict the %s module to the given organisation.', $module);
$setting['value'] = 0;
$setting['test'] = 'testLocalOrg';
$setting['type'] = 'numeric';
$setting['optionsSource'] = 'LocalOrgs';
} else {
$setting['test'] = 'testForEmpty';
$setting['type'] = 'string';
$setting['description'] = __('Set this required module specific setting.');
$setting['value'] = '';
}
$serverSettings['Plugin']['Enrichment_' . $module . '_' . $result['name']] = $setting;
}
}
$this->readModuleSettings($serverSettings, array('Enrichment'));
}
$finalSettingsUnsorted = array();
foreach ($serverSettings as $branchKey => &$branchValue) {
@ -3181,16 +3118,8 @@ class Server extends AppModel
public function runConnectionTest($id)
{
$server = $this->find('first', array('conditions' => array('Server.id' => $id)));
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
$HttpSocket = $this->setupHttpSocket($server);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/servers/getVersion';
try {
$response = $HttpSocket->get($uri, false, $request);
@ -3233,16 +3162,8 @@ class Server extends AppModel
public function runPOSTtest($id)
{
$server = $this->find('first', array('conditions' => array('Server.id' => $id)));
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
$HttpSocket = $this->setupHttpSocket($server);
$request = $this->setupSyncRequest($server);
$testFile = file_get_contents(APP . 'files/scripts/test_payload.txt');
$uri = $server['Server']['url'] . '/servers/postTest';
$this->Log = ClassRegistry::init('Log');
@ -3307,19 +3228,9 @@ class Server extends AppModel
$localVersion = json_decode($file->read(), true);
$file->close();
$server = $this->find('first', array('conditions' => array('Server.id' => $id)));
if (!$HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/servers/getVersion';
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
try {
$response = $HttpSocket->get($uri, '', $request);
} catch (Exception $e) {
@ -3407,41 +3318,6 @@ class Server extends AppModel
return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'version' => $remoteVersion);
}
/* This is a fallback for legacy remote instances that don't report back the current user's sync permission.
*
* The idea is simple: If we have no way of determining the perm_sync flag from the remote instance, request
* /servers/testConnection from the remote. This API is used to check the remote connectivity and expects an ID to be passed
* In this case however we are not passing an ID so ideally it will return 404, meaning that the instance is invalid.
* We are abusing the fact that only sync users can use this functionality, if we don't have sync permission we'll get a 403
* instead of the 404. It's hacky but it works fine and serves the purpose.
*/
public function checkLegacyServerSyncPrivilege($id, $HttpSocket = false)
{
$server = $this->find('first', array('conditions' => array('Server.id' => $id)));
if (!$HttpSocket) {
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
}
$uri = $server['Server']['url'] . '/servers/testConnection';
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
)
);
try {
$response = $HttpSocket->get($uri, '', $request);
} catch (Exception $e) {
return false;
}
if ($response->code == '404') {
return true;
}
return false;
}
public function isJson($string)
{
return (json_last_error() == JSON_ERROR_NONE);
@ -3782,30 +3658,10 @@ class Server extends AppModel
$worker = $workers[$pid];
if (substr_count(trim(shell_exec('ps -p ' . $pid)), PHP_EOL) > 0 ? true : false) {
shell_exec('kill ' . $pid . ' > /dev/null 2>&1 &');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'User',
'model_id' => $user['id'],
'email' => $user['email'],
'action' => 'stop_worker',
'user_id' => $user['id'],
'title' => 'Stopping a worker.',
'change' => 'Stopping a worker. Worker was of type ' . $worker['queue'] . ' with pid ' . $pid
));
$this->__logRemoveWorker($user, $pid, $worker['queue'], false);
} else {
$this->ResqueStatus->removeWorker($pid);
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'User',
'model_id' => $user['id'],
'email' => $user['email'],
'action' => 'remove_dead_workers',
'user_id' => $user['id'],
'title' => 'Removing a dead worker.',
'change' => 'Removing dead worker data. Worker was of type ' . $worker['queue'] . ' with pid ' . $pid
));
$this->__logRemoveWorker($user, $pid, $worker['queue'], true);
}
$this->ResqueStatus->removeWorker($pid);
}
@ -3815,7 +3671,6 @@ class Server extends AppModel
{
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$this->Log = ClassRegistry::init('Log');
if (function_exists('posix_getpwuid')) {
$currentUser = posix_getpwuid(posix_geteuid());
$currentUser = $currentUser['name'];
@ -3829,34 +3684,49 @@ class Server extends AppModel
$pidTest = substr_count(trim(shell_exec('ps -p ' . $pid)), PHP_EOL) > 0 ? true : false;
if ($worker['user'] == $currentUser && !$pidTest) {
$this->ResqueStatus->removeWorker($pid);
$this->Log->create();
if (!empty($user)) {
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'User',
'model_id' => $user['id'],
'email' => $user['email'],
'action' => 'remove_dead_workers',
'user_id' => $user['id'],
'title' => 'Removing a dead worker.',
'change' => 'Removing dead worker data. Worker was of type ' . $worker['queue'] . ' with pid ' . $pid
));
} else {
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'User',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'remove_dead_workers',
'user_id' => 0,
'title' => 'Removing a dead worker.',
'change' => 'Removing dead worker data. Worker was of type ' . $worker['queue'] . ' with pid ' . $pid
));
}
$this->__logRemoveWorker($user, $pid, $worker['queue'], true);
}
}
}
private function __logRemoveWorker($user, $pid, $queue, $dead = false)
{
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
if (empty($user)) {
$user = array(
'id' => 0,
'Organisation' => array(
'name' => 'SYSTEM'
),
'email' => 'SYSTEM'
);
}
$type = $dead ? 'dead' : 'kill';
$text = array(
'dead' => array(
'action' => 'remove_dead_workers',
'title' => __('Removing a dead worker.'),
'change' => sprintf(__('Removing dead worker data. Worker was of type %s with pid %s'), $queue, $pid)
),
'kill' => array(
'action' => 'stop_worker',
'title' => __('Stopping a worker.'),
'change' => sprintf(__('Stopping a worker. Worker was of type %s with pid %s'), $queue, $pid)
)
);
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'User',
'model_id' => $user['id'],
'email' => $user['email'],
'action' => $text[$type]['action'],
'user_id' => $user['id'],
'title' => $text[$type]['title'],
'change' => $text[$type]['change']
));
}
public function upgrade2324($user_id, $jobId = false)
{
$this->cleanCacheFiles();
@ -4055,17 +3925,8 @@ class Server extends AppModel
if (empty($server)) {
return 2;
}
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
//'Connection' => 'keep-alive' // // LATER followup cakephp issue about this problem: https://github.com/cakephp/cakephp/issues/1961
)
);
$HttpSocket = $this->setupHttpSocket($server);
$request = $this->setupSyncRequest($server);
$validArgs = array_merge(array('sort', 'direction'), $this->validEventIndexFilters);
$urlParams = '';
foreach ($validArgs as $v) {
@ -4115,17 +3976,8 @@ class Server extends AppModel
if (empty($server)) {
return 2;
}
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$request = array(
'header' => array(
'Authorization' => $server['Server']['authkey'],
'Accept' => 'application/json',
'Content-Type' => 'application/json',
//'Connection' => 'keep-alive' // // LATER followup cakephp issue about this problem: https://github.com/cakephp/cakephp/issues/1961
)
);
$HttpSocket = $this->setupHttpSocket($server);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/events/' . $eventId;
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->code == 200) {

33
app/Model/Tag.php
View File

@ -174,6 +174,39 @@ class Tag extends AppModel
return array($acceptIds, $rejectIds);
}
// find all of the tag Ids that belong to the accepted tags and the rejected tags
public function fetchTagIds($accept = array(), $reject = array())
{
$acceptIds = array();
$rejectIds = array();
if (!empty($accept)) {
$acceptIds = $this->findTagIdsByTagNames($accept);
if (empty($acceptIds)) {
$acceptIds[] = -1;
}
}
if (!empty($reject)) {
$rejectIds = $this->findTagIdsByTagNames($reject);
}
return array($acceptIds, $rejectIds);
}
// pass a list of tag names to receive a list of matched tag IDs
public function findTagIdsByTagNames($array)
{
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(Tag.name) like' => strtolower($a));
}
$params = array(
'recursive' => 1,
'conditions' => $conditions,
'fields' => array('Tag.id', 'Tag.id')
);
$result = $this->find('list', $params);
return array_values($result);
}
public function findEventIdsByTagNames($array)
{
$ids = array();

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 4cf84858e31b376438f7855f69c07552d80fff1f
Subproject commit a0dfdd65ae2aeab3e9552535cd576c7399694e88