MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/4584/head
chrisr3d 2019-04-12 15:25:22 +02:00
parent 58129260f9 99508fcf6b
commit c527077b1c
27 changed files with 3203 additions and 2809 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PyMISP

@ -1 +1 @@
Subproject commit a68bd80ab9dceaee9674bd9a2b0bffc4f387fcdc
Subproject commit e8334be9caaddab229d850c92cb029812222dd82

467
app/Console/Command/AdminShell.php
View File

@ -2,173 +2,281 @@
App::uses('AppShell', 'Console/Command');
class AdminShell extends AppShell
{
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role');
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role');
public function jobGenerateCorrelation() {
$jobId = $this->args[0];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Attribute');
$this->Attribute->generateCorrelation($jobId, 0);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function jobGenerateCorrelation() {
$jobId = $this->args[0];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Attribute');
$this->Attribute->generateCorrelation($jobId, 0);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function jobPurgeCorrelation() {
$jobId = $this->args[0];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Attribute');
$this->Attribute->purgeCorrelations();
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function jobPurgeCorrelation() {
$jobId = $this->args[0];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Attribute');
$this->Attribute->purgeCorrelations();
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function jobGenerateShadowAttributeCorrelation() {
$jobId = $this->args[0];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('ShadowAttribute');
$this->ShadowAttribute->generateCorrelation($jobId);
}
public function jobGenerateShadowAttributeCorrelation() {
$jobId = $this->args[0];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('ShadowAttribute');
$this->ShadowAttribute->generateCorrelation($jobId);
}
public function updateMISP() {
$status = array('branch' => '2.4');
echo $this->Server->update($status) . PHP_EOL;
}
public function restartWorkers() {
public function restartWorkers()
{
$this->Server->restartWorkers();
echo PHP_EOL . 'Workers restarted.' . PHP_EOL;
}
public function updateGalaxies() {
// The following is 7.x upwards only
//$value = $this->args[0] ?? $this->args[0] ?? 0;
$value = empty($this->args[0]) ? null : $this->args[0];
if ($value === 'false') $value = 0;
if ($value === 'true') $value = 1;
if ($value === 'force') $value = 1;
$force = $value;
$result = $this->Galaxy->update($force);
if ($result) {
echo 'Galaxies updated';
} else {
echo 'Could not update Galaxies';
}
}
public function updateAfterPull() {
$this->loadModel('Job');
$this->loadModel('Server');
$submodule_name = $this->args[0];
$jobId = $this->args[1];
$userId = $this->args[2];
$this->Job->id = $jobId;
$result = $this->Server->updateAfterPull($submodule_name, $userId);
$this->Job->saveField('progress', 100);
$this->Job->saveField('date_modified', date("y-m-d H:i:s"));
if ($result) {
$this->Job->saveField('message', __('Database updated: ' . $submodule_name));
} else {
$this->Job->saveField('message', __('Could not update the database: ' . $submodule_name));
}
}
# FIXME: Make Taxonomy->update() return a status string on API if successful
public function updateTaxonomies() {
$result = $this->Taxonomy->update();
if ($result) {
echo 'Taxonomies updated';
} else {
echo 'Could not update Taxonomies';
}
}
public function restartWorker()
{
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
echo 'Usage: ' . APP . '/cake ' . 'Admin restartWorker [PID]';
}
$pid = $this->args[0];
$result = $this->Server->restartWorker($pid);
if ($result === true) {
$response = __('Worker restarted.');
} else {
$response = __('Could not restart the worker. Reason: %s', $result);
}
echo sprintf(
'%s%s%s',
PHP_EOL,
$response,
PHP_EOL
);
}
public function updateWarningLists() {
$result = $this->Galaxy->update();
if ($result) {
echo 'Warning lists updated';
} else {
echo 'Could not update warning lists';
}
}
public function killWorker()
{
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
echo 'Usage: ' . APP . '/cake ' . 'Admin killWorker [PID]';
die();
}
$pid = $this->args[0];
$result = $this->Server->killWorker($pid, false);
echo sprintf(
'%s%s%s',
PHP_EOL,
__('Worker killed.'),
PHP_EOL
);
}
public function updateNoticeLists() {
$result = $this->Noticelist->update();
if ($result) {
echo 'Notice lists updated';
} else {
echo 'Could not update notice lists';
}
}
public function startWorker()
{
if (empty($this->args[0])) {
echo 'Usage: ' . APP . '/cake ' . 'Admin startWorker [queue]';
die();
}
$queue = $this->args[0];
$this->Server->startWorker($queue);
echo sprintf(
'%s%s%s',
PHP_EOL,
__('Worker started.'),
PHP_EOL
);
}
# FIXME: Debug and make it work, fails to pass userId/orgId properly
public function updateObjectTemplates() {
if (empty($this->args[0])) {
echo 'Usage: ' . APP . '/cake ' . 'Admin updateNoticeLists [user_id]';
} else {
$userId = $this->args[0];
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array(
'User.id' => $userId,
),
'fields' => array('User.id', 'User.org_id')
));
if (empty($user)) {
echo 'User not found';
} else {
$result = $this->ObjectTemplate->update($user, false,false);
if ($result) {
echo 'Object templates updated';
} else {
echo 'Could not update object templates';
}
}
}
}
public function updateJSON() {
echo 'Updating all JSON structures.' . PHP_EOL;
$results = $this->Server->updateJSON();
foreach ($results as $type => $result) {
if ($result !== false) {
echo sprintf(
__('%s updated.') . PHP_EOL,
Inflector::pluralize(Inflector::humanize($type))
);
} else {
echo sprintf(
__('Could not update %s.') . PHP_EOL,
Inflector::pluralize(Inflector::humanize($type))
);
}
}
echo 'All JSON structures updated. Thank you and have a very safe and productive day.' . PHP_EOL;
}
public function jobUpgrade24() {
$jobId = $this->args[0];
$user_id = $this->args[1];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Server');
$this->Server->upgrade2324($user_id, $jobId);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function updateGalaxies() {
// The following is 7.x upwards only
//$value = $this->args[0] ?? $this->args[0] ?? 0;
$value = empty($this->args[0]) ? null : $this->args[0];
if ($value === 'false') $value = 0;
if ($value === 'true') $value = 1;
if ($value === 'force') $value = 1;
$force = $value;
$result = $this->Galaxy->update($force);
if ($result) {
echo 'Galaxies updated';
} else {
echo 'Could not update Galaxies';
}
}
public function prune_update_logs() {
$jobId = $this->args[0];
$user_id = $this->args[1];
$user = $this->User->getAuthUser($user_id);
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Log');
$this->Log->pruneUpdateLogs($jobId, $user);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
# FIXME: Make Taxonomy->update() return a status string on API if successful
public function updateTaxonomies() {
$result = $this->Taxonomy->update();
if ($result) {
echo 'Taxonomies updated';
} else {
echo 'Could not update Taxonomies';
}
}
public function getWorkers() {
$result = $this->Server->workerDiagnostics($workerIssueCount);
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
public function updateWarningLists() {
$result = $this->Galaxy->update();
if ($result) {
echo 'Warning lists updated';
} else {
echo 'Could not update warning lists';
}
}
public function updateNoticeLists() {
$result = $this->Noticelist->update();
if ($result) {
echo 'Notice lists updated';
} else {
echo 'Could not update notice lists';
}
}
# FIXME: Debug and make it work, fails to pass userId/orgId properly
public function updateObjectTemplates() {
if (empty($this->args[0])) {
echo 'Usage: ' . APP . '/cake ' . 'Admin updateNoticeLists [user_id]';
} else {
$userId = $this->args[0];
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array(
'User.id' => $userId,
),
'fields' => array('User.id', 'User.org_id')
));
if (empty($user)) {
echo 'User not found';
} else {
$result = $this->ObjectTemplate->update($user, false,false);
if ($result) {
echo 'Object templates updated';
} else {
echo 'Could not update object templates';
}
}
}
}
public function jobUpgrade24() {
$jobId = $this->args[0];
$user_id = $this->args[1];
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Server');
$this->Server->upgrade2324($user_id, $jobId);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function prune_update_logs() {
$jobId = $this->args[0];
$user_id = $this->args[1];
$user = $this->User->getAuthUser($user_id);
$this->loadModel('Job');
$this->Job->id = $jobId;
$this->loadModel('Log');
$this->Log->pruneUpdateLogs($jobId, $user);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
$this->Job->saveField('status', 4);
}
public function getWorkers() {
$result = $this->Server->workerDiagnostics($workerIssueCount);
$query = 'all';
if (!empty($this->args[0])) {
$query = $this->args[0];
}
if ($query === 'dead') {
$dead_workers = array();
foreach ($result as $queue => $data) {
if (!empty($data['workers'])) {
foreach ($data['workers'] as $k => $worker) {
if ($worker['alive']) {
unset($result[$queue]['workers'][$k]);
}
}
}
if (empty($result[$queue]['workers'])) {
unset($result[$queue]);
}
}
}
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
}
public function getSetting() {
$param = empty($this->args[0]) ? 'all' : $this->args[0];
$settings = $this->Server->serverSettingsRead();
$result = $settings;
if (!empty($param)) {
$result = 'No valid setting found for ' . $param;
foreach ($settings as $setting) {
if ($setting['setting'] == $param) {
$result = $setting;
break;
}
}
}
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
}
public function getSetting() {
$param = empty($this->args[0]) ? 'all' : $this->args[0];
$settings = $this->Server->serverSettingsRead();
$result = $settings;
if (!empty($param)) {
$result = 'No valid setting found for ' . $param;
foreach ($settings as $setting) {
if ($setting['setting'] == $param) {
$result = $setting;
break;
}
}
}
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
}
public function setSetting() {
$setting_name = !isset($this->args[0]) ? null : $this->args[0];
$value = !isset($this->args[1]) ? null : $this->args[1];
if ($value === 'false') $value = 0;
if ($value === 'true') $value = 1;
public function setSetting() {
$setting_name = !isset($this->args[0]) ? null : $this->args[0];
$value = !isset($this->args[1]) ? null : $this->args[1];
if ($value === 'false') $value = 0;
if ($value === 'true') $value = 1;
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || $value === null) {
echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]';
} else {
if (empty($setting_name) || $value === null) {
echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]';
} else {
$setting = $this->Server->getSettingData($setting_name);
if (empty($setting)) {
echo 'Invalid setting. Please make sure that the setting that you are attempting to change exists.';
@ -179,25 +287,25 @@ class AdminShell extends AppShell
} else {
echo $result;
}
}
}
echo PHP_EOL;
}
}
public function setDatabaseVersion() {
if (empty($this->args[0])) echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setDatabaseVersion [db_version]' . PHP_EOL;
else {
$db_version = $this->AdminSetting->find('first', array(
'conditions' => array('setting' => 'db_version')
));
if (!empty($db_version)) {
$db_version['value'] = trim($this->args[0]);
$this->AdminSetting->save($db_version);
echo 'Database version set. MISP will replay all of the upgrade scripts since the selected version on the next user login.' . PHP_EOL;
} else {
echo 'Something went wrong. Could not find the existing db version.' . PHP_EOL;
}
}
}
public function setDatabaseVersion() {
if (empty($this->args[0])) echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setDatabaseVersion [db_version]' . PHP_EOL;
else {
$db_version = $this->AdminSetting->find('first', array(
'conditions' => array('setting' => 'db_version')
));
if (!empty($db_version)) {
$db_version['value'] = trim($this->args[0]);
$this->AdminSetting->save($db_version);
echo 'Database version set. MISP will replay all of the upgrade scripts since the selected version on the next user login.' . PHP_EOL;
} else {
echo 'Something went wrong. Could not find the existing db version.' . PHP_EOL;
}
}
}
public function updateDatabase() {
echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
@ -222,20 +330,20 @@ class AdminShell extends AppShell
}
}
public function clearBruteforce()
{
$conditions = array('Bruteforce.username !=' => '');
if (!empty($this->args[0])) {
public function clearBruteforce()
{
$conditions = array('Bruteforce.username !=' => '');
if (!empty($this->args[0])) {
$conditions = array('Bruteforce.username' => $this->args[0]);
}
$result = $this->Bruteforce->deleteAll($conditions, false, false);
$target = empty($this->args[0]) ? 'all users' : $this->args[0];
if ($result) {
echo 'Brutefoce entries for ' . $target . ' deleted.' . PHP_EOL;
} else {
echo 'Something went wrong, could not delete bruteforce entries for ' . $target . '.' . PHP_EOL;
}
}
$result = $this->Bruteforce->deleteAll($conditions, false, false);
$target = empty($this->args[0]) ? 'all users' : $this->args[0];
if ($result) {
echo 'Brutefoce entries for ' . $target . ' deleted.' . PHP_EOL;
} else {
echo 'Something went wrong, could not delete bruteforce entries for ' . $target . '.' . PHP_EOL;
}
}
public function setDefaultRole()
{
@ -296,4 +404,17 @@ class AdminShell extends AppShell
}
echo 'Updated, new key:' . PHP_EOL . $authKey . PHP_EOL;
}
public function getOptionParser() {
$parser = parent::getOptionParser();
$parser->addSubcommand('updateJSON', array(
'help' => __('Update the JSON definitions of MISP.'),
'parser' => array(
'arguments' => array(
'update' => array('help' => __('Update the submodules before ingestion.'), 'short' => 'u', 'boolean' => 1)
)
)
));
return $parser;
}
}

59
app/Console/worker/start.sh
View File

@ -1,57 +1,18 @@
#!/usr/bin/env bash
# TODO: Put some logic inside if many worker PIDs are detected
# Extract base directory where this script is and cd into it
cd "${0%/*}"
# Set to the current webroot owner
WWW_USER=$(ls -l ../cake |awk {'print $3'}|tail -1)
# In most cases the owner of the cake script is also the user as which it should be executed.
if [[ "$USER" != "$WWW_USER" ]]; then
echo "You run this script as $USER and the owner of the cake command is $WWW_USER. This might be an issue."
fi
# Check if run as root
if [[ "$EUID" -eq "0" ]]; then
if [ "$EUID" -eq 0 ]; then
echo "Please DO NOT run the worker script as root"
exit 1
fi
# Check if jq is present and enable advanced checks
if [[ "$(jq -V > /dev/null 2> /dev/null; echo $?)" != 0 ]]; then
echo "jq is not installed, disabling advanced checks."
ADVANCED="0"
else
ADVANCED="1"
fi
# Extract base directory where this script is and cd into it
cd "${0%/*}"
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque startscheduler --interval 5
if [[ "$ADVANCED" == "1" ]]; then
for worker in `echo cache default email prio scheduler`; do
workerStatus=$(../cake Admin getWorkers |tail -n +7 |jq -r ".$worker" |jq -r '.ok')
PIDcount=$(../cake admin getWorkers |tail -n +7 |jq -r ".$worker.workers" |grep pid | wc -l)
echo -n "$worker has $PIDcount PID(s)"
if [[ "$workerStatus" != "true" ]]; then
echo ", trying to restart."
if [[ "$worker" != "scheduler" ]]; then
../cake CakeResque.CakeResque start --interval 5 --queue $worker
else
../cake CakeResque.CakeResque startscheduler --interval 5
fi
else
echo ", up and running."
fi
done
exit 0
else
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque startscheduler --interval 5
exit 0
fi
exit 0

57
app/Console/worker/start_dynamic.sh Normal file
View File

@ -0,0 +1,57 @@
#!/usr/bin/env bash
# TODO: Put some logic inside if many worker PIDs are detected
# Extract base directory where this script is and cd into it
cd "${0%/*}"
# Set to the current webroot owner
WWW_USER=$(ls -l ../cake |awk {'print $3'}|tail -1)
# In most cases the owner of the cake script is also the user as which it should be executed.
if [[ "$USER" != "$WWW_USER" ]]; then
echo "You run this script as $USER and the owner of the cake command is $WWW_USER. This might be an issue."
fi
# Check if run as root
if [[ "$EUID" -eq "0" ]]; then
echo "Please DO NOT run the worker script as root"
exit 1
fi
# Check if jq is present and enable advanced checks
if [[ "$(jq -V > /dev/null 2> /dev/null; echo $?)" != 0 ]]; then
echo "jq is not installed, disabling advanced checks."
ADVANCED="0"
else
ADVANCED="1"
fi
if [[ "$ADVANCED" == "1" ]]; then
for worker in `echo cache default email prio scheduler`; do
workerStatus=$(../cake Admin getWorkers |tail -n +7 |jq -r ".$worker" |jq -r '.ok')
PIDcount=$(../cake admin getWorkers |tail -n +7 |jq -r ".$worker.workers" |grep pid | wc -l)
echo -n "$worker has $PIDcount PID(s)"
if [[ "$workerStatus" != "true" ]]; then
echo ", trying to restart."
if [[ "$worker" != "scheduler" ]]; then
../cake CakeResque.CakeResque start --interval 5 --queue $worker
else
../cake CakeResque.CakeResque startscheduler --interval 5
fi
else
echo ", up and running."
fi
done
exit 0
else
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque startscheduler --interval 5
exit 0
fi

52
app/Controller/AttributesController.php
View File

@ -1197,7 +1197,7 @@ class AttributesController extends AppController
}
}
public function viewPicture($id, $thumbnail=false, $width=200, $height=200)
public function viewPicture($id, $thumbnail=false)
{
if (Validation::uuid($id)) {
$temp = $this->Attribute->find('first', array(
@ -1221,11 +1221,15 @@ class AttributesController extends AppController
'Attribute.id' => $id,
'Attribute.type' => 'attachment'
),
'withAttachments' => true,
'includeAllTags' => false,
'includeAttributeUuid' => true,
'flatten' => true
);
if ($this->_isRest()) {
$conditions['withAttachments'] = true;
}
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $conditions);
if (empty($attribute)) {
throw new MethodNotAllowedException('Invalid attribute');
@ -1235,49 +1239,11 @@ class AttributesController extends AppController
if ($this->_isRest()) {
return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type());
} else {
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200;
$image_data = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height);
$extension = explode('.', $attribute['Attribute']['value']);
$extension = end($extension);
if (extension_loaded('gd')) {
$image = ImageCreateFromString(base64_decode($attribute['Attribute']['data']));
if (!$thumbnail) {
ob_start ();
switch ($extension) {
case 'gif':
imagegif($image);
break;
case 'jpg':
case 'jpeg':
imagejpeg($image);
break;
case 'png':
imagepng($image);
break;
default:
break;
}
$image_data = $extension != 'gif' ? ob_get_contents() : base64_decode($attribute['Attribute']['data']);
ob_end_clean ();
imagedestroy($image);
} else { // thumbnail requested, resample picture with desired dimension
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 150;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 150;
if ($extension == 'gif') {
$image_data = base64_decode($attribute['Attribute']['data']);
} else {
$extension = 'jpg';
$imageTC = ImageCreateTrueColor($width, $height);
ImageCopyResampled($imageTC, $image, 0, 0, 0, 0, $width, $height, ImageSX($image), ImageSY($image));
ob_start ();
imagejpeg ($imageTC);
$image_data = ob_get_contents();
ob_end_clean ();
imagedestroy($image);
imagedestroy($imageTC);
}
}
} else {
$image_data = base64_decode($attribute['Attribute']['data']);
}
$this->response->type(strtolower(h($extension)));
$this->response->body($image_data);
$this->autoRender = false;

1
app/Controller/Component/ACLComponent.php
View File

@ -358,6 +358,7 @@ class ACLComponent extends Component
'stopZeroMQServer' => array(),
'testConnection' => array('perm_sync'),
'update' => array(),
'updateJSON' => array(),
'updateSubmodule' => array(),
'uploadFile' => array(),
'clearWorkerQueue' => array()

2
app/Controller/Component/RestResponseComponent.php
View File

@ -1552,7 +1552,7 @@ class RestResponseComponent extends Component
$field['values'] = $tags;
}
private function __overwriteNationality($scope, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Organisation")->countries);
$field['values'] = ClassRegistry::init("Organisation")->countries;
}
private function __overwriteAction($scope, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Log")->actionDefinitions);

8
app/Controller/ServersController.php
View File

@ -1555,7 +1555,7 @@ class ServersController extends AppController
if ($this->request->is('post')) {
$request = $this->request->data;
$submodule = $request['Server']['submodule'];
$res = $this->Server->updateSubmodule($submodule);
$res = $this->Server->updateSubmodule($this->Auth->user(), $submodule);
return new CakeResponse(array('body'=> json_encode($res), 'type' => 'json'));
} else {
throw new MethodNotAllowedException();
@ -1832,4 +1832,10 @@ misp.direct_call(relative_path, body)
$this->redirect(array('action' => 'index'));
}
}
public function updateJSON()
{
$results = $this->Server->updateJSON();
return $this->RestResponse->viewData($results, $this->response->type());
}
}

1
app/Controller/TaxonomiesController.php
View File

@ -60,6 +60,7 @@ class TaxonomiesController extends AppController
$this->loadModel('AttributeTag');
foreach ($taxonomy['entries'] as $key => $value) {
$count = 0;
$count_a = 0;
if (!empty($value['existing_tag'])) {
foreach ($value['existing_tag'] as $et) {
$count = $this->EventTag->find('count', array(

81
app/Lib/Tools/ComplexTypeTool.php
View File

@ -3,20 +3,73 @@
class ComplexTypeTool
{
private $__refangRegexTable = array(
'/^hxxp/i' => 'http',
'/^meow/i' => 'http',
'/^h\[tt\]p/i' => 'http',
'/\[\.\]/' => '.',
'/\[dot\]/' => '.',
'/\(dot\)/' => '.',
'/\\\\\./' => '.',
'/\.+/' => '.',
'/\[hxxp:\/\/\]/' => 'http://',
'/\\\/' => '',
'/[\@]/' => '@',
'/\[:\]/' => ':'
array(
'from' => '/^hxxp/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'from' => '/^meow/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'from' => '/^h\[tt\]p/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'from' => '/\[\.\]/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\[dot\]/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\(dot\)/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\\\\\./',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\.+/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\[hxxp:\/\/\]/',
'to' => 'http://',
'types' => array('link', 'url')
),
array(
'from' => '/[\@]/',
'to' => '@',
'types' => array('email-src', 'email-dst')
),
array(
'from' => '/\[:\]/',
'to' => ':',
'types' => array('url', 'link')
)
);
public function refangValue($value, $type)
{
foreach ($this->__refangRegexTable as $regex) {
if (!isset($regex['types']) || in_array($type, $regex['types'])) {
$value = preg_replace($regex['from'], $regex['to'], $value);
}
}
return $value;
}
private $__tlds = array();
public function setTLDs($tlds = array())
@ -329,8 +382,8 @@ class ComplexTypeTool
private function __refangInput($input)
{
$input['refanged'] = $input['raw'];
foreach ($this->__refangRegexTable as $regex => $replacement) {
$input['refanged'] = preg_replace($regex, $replacement, $input['refanged']);
foreach ($this->__refangRegexTable as $regex) {
$input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']);
}
$input['refanged'] = rtrim($input['refanged'], ".");
$input['refanged'] = preg_replace_callback(

97
app/Model/Attribute.php
View File

@ -774,6 +774,10 @@ class Attribute extends AppModel
if (is_array($this->data['Attribute']['value'])) {
return false;
}
App::uses('ComplexTypeTool', 'Tools');
$this->complexTypeTool = new ComplexTypeTool();
$this->data['Attribute']['value'] = $this->complexTypeTool->refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
return false;
@ -1585,7 +1589,7 @@ class Attribute extends AppModel
}
}
public function base64EncodeAttachment($attribute)
public function getAttachment($attribute, $path_suffix='')
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
@ -1596,21 +1600,20 @@ class Attribute extends AppModel
// S3 - we have to first get the object then we can encode it
$s3 = $this->getS3Client();
// This will return the content of the object
$content = $s3->download($attribute['event_id'] . DS . $attribute['id']);
$content = $s3->download($attribute['event_id'] . DS . $attribute['id'] . $path_suffix);
} else {
// Standard filesystem
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
$file = new File($filepath);
if (!$file->readable()) {
return '';
}
$content = $file->read();
}
return base64_encode($content);
return $content;
}
public function saveBase64EncodedAttachment($attribute)
public function saveAttachment($attribute, $path_suffix='')
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
@ -1622,17 +1625,17 @@ class Attribute extends AppModel
// We don't need your fancy directory structures and
// PEE AICH PEE meddling
$s3 = $this->getS3Client();
$data = base64_decode($attribute['data']);
$key = $attribute['event_id'] . DS . $attribute['id'];
$data = $attribute['data'];
$key = $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
$s3->upload($key, $data);
return true;
} else {
// Plebian filesystem operations
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$destpath = $rootDir . DS . $attribute['id'] . $path_suffix;
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
$decodedData = $attribute['data']; // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
@ -1642,6 +1645,73 @@ class Attribute extends AppModel
}
}
public function base64EncodeAttachment($attribute)
{
return base64_encode($this->getAttachment($attribute));
}
public function saveBase64EncodedAttachment($attribute)
{
$attribute['data'] = base64_decode($attribute['data']);
return $this->saveAttachment($attribute);
}
public function getPictureData($attribute, $thumbnail=false, $width=200, $height=200)
{
$extension = explode('.', $attribute['Attribute']['value']);
$extension = end($extension);
if (extension_loaded('gd')) {
if (!$thumbnail) {
$data = $this->getAttachment($attribute['Attribute']);
$image = ImageCreateFromString($data);
ob_start ();
switch ($extension) {
case 'gif':
// php-gd doesn't support animated gif. Skipping...
break;
case 'jpg':
case 'jpeg':
imagejpeg($image);
break;
case 'png':
imagepng($image);
break;
default:
break;
}
$image_data = $extension != 'gif' ? ob_get_contents() : $data;
ob_end_clean ();
} else { // thumbnail requested, resample picture with desired dimension and save result
$thumbnail_exists = $this->getAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
if ($width == 200 && $height == 200 && $thumbnail_exists !== '') { // check if thumbnail already exists
$image_data = $thumbnail_exists;
} else {
$data = $this->getAttachment($attribute['Attribute']);
if ($extension == 'gif') {
$image_data = $data;
} else {
$image = ImageCreateFromString($data);
$extension = 'jpg';
$imageTC = ImageCreateTrueColor($width, $height);
ImageCopyResampled($imageTC, $image, 0, 0, 0, 0, $width, $height, ImageSX($image), ImageSY($image));
ob_start ();
imagejpeg ($imageTC);
$image_data = ob_get_contents();
ob_end_clean ();
imagedestroy($image);
imagedestroy($imageTC);
}
// save thumbnail for later reuse
$attribute['Attribute']['data'] = $image_data;
$this->saveAttachment($attribute['Attribute'], '_thumbnail');
}
}
} else {
$image_data = $this->getAttachment($attribute['Attribute']);
}
return $image_data;
}
public function __beforeSaveCorrelation($a)
{
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
@ -3527,13 +3597,6 @@ class Attribute extends AppModel
if ($attribute['value'] == $tmpfile->name) {
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename;
}
if (!empty($attribute['encrypt'])) {
if (!empty($attribute['encrypt']) && $attribute['encrypt']) {
$encrypted = $this->handleMaliciousBase64($event_id, $filename, $attribute['data'], array('md5'));
$result['Object'][$k]['Attribute'][$k2]['data'] = $encrypted['data'];
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename . '|' . $encrypted['md5'];
}
}
}
}
}

3
app/Model/Event.php
View File

@ -1647,7 +1647,6 @@ class Event extends AppModel
'last',
'to_ids',
'includeAllTags',
'withAttachments',
'includeAttachments',
'event_uuid',
'distribution',
@ -3841,7 +3840,7 @@ class Event extends AppModel
}
}
private function __getPrioWorkerIfPossible()
public function __getPrioWorkerIfPossible()
{
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();

13
app/Model/ObjectTemplate.php
View File

@ -47,7 +47,7 @@ class ObjectTemplate extends AppModel
return true;
}
public function update($user, $type = false, $force = false)
public function update($user = false, $type = false, $force = false)
{
$objectsDir = APP . 'files/misp-objects/objects';
$directories = glob($objectsDir . '/*', GLOB_ONLYDIR);
@ -94,7 +94,7 @@ class ObjectTemplate extends AppModel
return $updated;
}
private function __updateObjectTemplate($template, $current, $user)
private function __updateObjectTemplate($template, $current, $user = false)
{
$success = false;
$template['requirements'] = array();
@ -104,8 +104,13 @@ class ObjectTemplate extends AppModel
$template['requirements'][$field] = $template[$field];
}
}
$template['user_id'] = $user['id'];
$template['org_id'] = $user['org_id'];
if (!empty($user)) {
$template['user_id'] = $user['id'];
$template['org_id'] = $user['org_id'];
} else {
$template['user_id'] = 0;
$template['org_id'] = 0;
}
$template['fixed'] = 1;
$this->create();
$result = $this->save($template);

160
app/Model/Server.php
View File

@ -105,33 +105,44 @@ class Server extends AppModel
$this->command_line_functions = array(
'console_admin_tasks' => array(
'data' => array(
'getSettings' => 'MISP/app/Console/cake Admin getSetting [setting]',
'setSettings' => 'MISP/app/Console/cake Admin setSetting [setting] [value]',
'getAuthkey' => 'MISP/app/Console/cake Admin getAuthkey [email]',
'setBaseurl' => 'MISP/app/Console/cake Baseurl [baseurl]',
'changePassword' => 'MISP/app/Console/cake Password [email] [new_password] [--override_password_change]',
'clearBruteforce' => 'MISP/app/Console/cake Admin clearBruteforce [user_email]',
'updateDatabase' => 'MISP/app/Console/cake Admin updateDatabase',
'updateGalaxies' => 'MISP/app/Console/cake Admin updateGalaxies',
'updateTaxonomies' => 'MISP/app/Console/cake Admin updateTaxonomies',
'updateObjectTemplates' => 'MISP/app/Console/cake Admin updateObjectTemplates',
'updateWarningLists' => 'MISP/app/Console/cake Admin updateWarningLists',
'updateNoticeLists' => 'MISP/app/Console/cake Admin updateNoticeLists',
'setDefaultRole' => 'MISP/app/Console/cake Admin setDefaultRole [role_id]'
'Get setting' => 'MISP/app/Console/cake Admin getSetting [setting]',
'Set setting' => 'MISP/app/Console/cake Admin setSetting [setting] [value]',
'Get authkey' => 'MISP/app/Console/cake Admin getAuthkey [email]',
'Set baseurl' => 'MISP/app/Console/cake Baseurl [baseurl]',
'Change password' => 'MISP/app/Console/cake Password [email] [new_password] [--override_password_change]',
'Clear Bruteforce Entries' => 'MISP/app/Console/cake Admin clearBruteforce [user_email]',
'Run database update' => 'MISP/app/Console/cake Admin updateDatabase',
'Update all JSON structures' => 'MISP/app/Console/cake Admin updateJSON',
'Update Galaxy definitions' => 'MISP/app/Console/cake Admin updateGalaxies',
'Update taxonomy definitions' => 'MISP/app/Console/cake Admin updateTaxonomies',
'Update object templates' => 'MISP/app/Console/cake Admin updateObjectTemplates',
'Update Warninglists' => 'MISP/app/Console/cake Admin updateWarningLists',
'Update Noticelists' => 'MISP/app/Console/cake Admin updateNoticeLists',
'Set default role' => 'MISP/app/Console/cake Admin setDefaultRole [role_id]'
),
'description' => __('Certain administrative tasks are exposed to the API, these help with maintaining and configuring MISP in an automated way / via external tools.'),
'header' => __('Administering MISP via the CLI')
),
'console_automation_tasks' => array(
'data' => array(
'pull' => 'MISP/app/Console/cake Server pull [user_id] [server_id] [full|update]',
'push' => 'MISP/app/Console/cake Server push [user_id] [server_id]',
'cacheFeed' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]',
'fetchFeed' => 'MISP/app/Console/cake Server fetchFeed [user_id] [feed_id|all|csv|text|misp]',
'enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]'
'Pull' => 'MISP/app/Console/cake Server pull [user_id] [server_id] [full|update]',
'Push' => 'MISP/app/Console/cake Server push [user_id] [server_id]',
'Cache feeds for quick lookups' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]',
'Fetch feeds as local data' => 'MISP/app/Console/cake Server fetchFeed [user_id] [feed_id|all|csv|text|misp]',
'Run enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]'
),
'description' => __('If you would like to automate tasks such as caching feeds or pulling from server instances, you can do it using the following command line tools. Simply execute the given commands via the command line / create cron jobs easily out of them.'),
'header' => __('Automating certain console tasks')
),
'worker_management_tasks' => array(
'data' => array(
'Get list of workers' => 'MISP/app/Console/cake Admin getWorkers [all|dead]',
'Start a worker' => 'MISP/app/Console/cake Admin startWorker [queue_name]',
'Restart a worker' => 'MISP/app/Console/cake Admin restartWorker [worker_pid]',
'Kill a worker' => 'MISP/app/Console/cake Admin killWorker [worker_pid]',
),
'description' => __('The background workers can be managed via the CLI in addition to the UI / API management tools'),
'header' => __('Managing the background workers')
)
);
@ -4637,24 +4648,90 @@ class Server extends AppModel
return $status;
}
public function updateSubmodule($submodule_name=false) {
public function updateSubmodule($user, $submodule_name=false) {
$path = APP . '../';
if ($submodule_name == false) {
$command = sprintf('cd %s; git submodule update 2>&1', $path);
exec($command, $output, $return_code);
$output = implode("\n", $output);
$res = array('status' => ($return_code==0 ? true : false), 'output' => $output);
if ($return_code == 0) { // update all DB
$res = array_merge($res, $this->updateDatabaseAfterPullRouter($submodule_name, $user));
}
} else if ($this->_isAcceptedSubmodule($submodule_name)) {
$command = sprintf('cd %s; git submodule update -- %s 2>&1', $path, $submodule_name);
exec($command, $output, $return_code);
$output = implode("\n", $output);
$res = array('status' => ($return_code==0 ? true : false), 'output' => $output);
if ($return_code == 0) { // update DB if necessary
$res = array_merge($res, $this->updateDatabaseAfterPullRouter($submodule_name, $user));
}
} else {
$res = array('status' => false, 'output' => __('Invalid submodule.'));
$res = array('status' => false, 'output' => __('Invalid submodule.'), 'job_sent' => false, 'sync_result' => __('unknown'));
}
return $res;
}
public function updateDatabaseAfterPullRouter($submodule_name, $user) {
if (Configure::read('MISP.background_jobs')) {
$job = ClassRegistry::init('Job');
$job->create();
$eventModel = ClassRegistry::init('Event');
$data = array(
'worker' => $eventModel->__getPrioWorkerIfPossible(),
'job_type' => __('update_after_pull'),
'job_input' => __('Updating: ' . $submodule_name),
'status' => 0,
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => 'Update database after PULL.',
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'AdminShell',
array('updateAfterPull', $submodule_name, $jobId, $user['id']),
true
);
$job->saveField('process_id', $process_id);
return array('job_sent' => true, 'sync_result' => __('unknown'));
} else {
$result = $this->updateAfterPull($submodule_name, $user['id']);
return array('job_sent' => false, 'sync_result' => $result);
}
}
public function updateAfterPull($submodule_name, $userId) {
$user = $this->User->getAuthUser($userId);
$result = array();
if ($user['Role']['perm_site_admin']) {
$updateAll = empty($submodule_name);
if ($submodule_name == 'app/files/misp-galaxy' || $updateAll) {
$this->Galaxy = ClassRegistry::init('Galaxy');
$result[] = ($this->Galaxy->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL;
}
if ($submodule_name == 'app/files/misp-objects' || $updateAll) {
$this->ObjectTemplate = ClassRegistry::init('ObjectTemplate');
$result[] = ($this->ObjectTemplate->update($user, false, false) ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL;
}
if ($submodule_name == 'app/files/noticelists' || $updateAll) {
$this->Noticelist = ClassRegistry::init('Noticelist');
$result[] = ($this->Noticelist->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL;
}
if ($submodule_name == 'app/files/taxonomies' || $updateAll) {
$this->Taxonomy = ClassRegistry::init('Taxonomy');
$result[] = ($this->Taxonomy->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL;
}
if ($submodule_name == 'app/files/warninglists' || $updateAll) {
$this->Warninglist = ClassRegistry::init('Warninglist');
$result[] = ($this->Warninglist->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL;
}
}
return implode('\n', $result);
}
public function update($status)
{
$final = '';
@ -4709,6 +4786,37 @@ class Server extends AppModel
return true;
}
public function restartWorker($pid)
{
if (Configure::read('MISP.background_jobs')) {
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$pid = intval($pid);
if (!isset($workers[$pid])) {
return __('Invalid worker.');
}
$currentWorker = $workers[$pid];
$this->killWorker($pid, false);
$this->startWorker($currentWorker['queue']);
return true;
}
return __('Background workers not enabled.');
}
public function startWorker($queue)
{
$validTypes = array('default', 'email', 'scheduler', 'cache', 'prio');
if (!in_array($queue, $validTypes)) {
return __('Invalid worker type.');
}
if ($queue != 'scheduler') {
shell_exec(APP . 'Console' . DS . 'cake CakeResque.CakeResque start --interval 5 --queue ' . $queue .' > /dev/null 2>&1 &');
} else {
shell_exec(APP . 'Console' . DS . 'cake CakeResque.CakeResque startscheduler -i 5 > /dev/null 2>&1 &');
}
return true;
}
public function cacheServerInitiator($user, $id = 'all', $jobId = false)
{
$params = array(
@ -4810,4 +4918,16 @@ class Server extends AppModel
}
return $data;
}
public function updateJSON()
{
$toUpdate = array('Galaxy', 'Noticelist', 'Warninglist', 'Taxonomy', 'ObjectTemplate');
$results = array();
foreach ($toUpdate as $target) {
$this->$target = ClassRegistry::init($target);
$result = $this->$target->update();
$results[$target] = $result === false ? false : true;
}
return $results;
}
}

2
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -594,7 +594,7 @@
));
echo $this->element('/genericElements/SideMenu/side_menu_post_link', array(
'event_id' => 'deleteUser',
'url' => '/admin/delete/' . h($id),
'url' => '/admin/users/delete/' . h($id),
'text' => __('Delete User'),
'message' => __('Are you sure you want to delete # %s? It is highly recommended to never delete users but to disable them instead.', h($id))
));

6
app/View/Elements/generic_picker.ctp
View File

@ -3,7 +3,6 @@
* Generic select picker
*/
/** Config **/
$select_threshold = 7; // threshold above which pills will be replace by a select (unused if multiple is > 1)
$defaults_options = array(
'select_options' => array(
// 'multiple' => '', // set to add possibility to pick multiple options in the select
@ -19,6 +18,7 @@
'allow_single_deselect' => true,
),
'multiple' => 0,
'select_threshold' => 7, // threshold above which pills will be replace by a select (unused if multiple is > 1)
'functionName' => '', // function to be called on submit
'submitButtonText' => 'Submit',
'disabledSubmitButton' => false, // wether to not draw the submit button
@ -45,9 +45,9 @@
} else { // multiple enabled
$defaults['chosen_options']['max_selected_options'] = $defaults['multiple'] == -1 ? 'Infinity' : $defaults['multiple'];
$defaults['select_options']['multiple'] = '';
$select_threshold = 0;
$defaults['select_threshold'] = 0;
}
$use_select = count($items) > $select_threshold;
$use_select = count($items) > $defaults['select_threshold'];
$countThresholdReached = count($items) > 1000;
$option_templates = array();
$options_additionalData = array();

43
app/View/Elements/healthElements/diagnostics.ctp
View File

@ -61,7 +61,10 @@
<pre class="hidden green bold" id="gitResult"></pre>
<button title="<?php echo __('Pull the latest MISP version from github');?>" class="btn btn-inverse" style="padding-top:1px;padding-bottom:1px;" onClick = "updateMISP();"><?php echo __('Update MISP');?></button>
</div>
<h3><?php echo __('Submodules version');?><it id="refreshSubmoduleStatus" class="fas fa-sync useCursorPointer" style="font-size: small; margin-left: 5px;"></it></h3>
<h3><?php echo __('Submodules version');?>
<it id="refreshSubmoduleStatus" class="fas fa-sync useCursorPointer" style="font-size: small; margin-left: 5px;" title="<?php echo __('Refresh submodules version.'); ?>"></it>
<it id="updateAllJson" class="fas fa-file-upload useCursorPointer" style="font-size: small; margin-left: 5px;" title="<?php echo __('Load all JSON into the database.'); ?>"></it>
</h3>
<div id="divSubmoduleVersions" style="background-color:#f7f7f9;">
</div>
@ -365,16 +368,50 @@
<script>
$(document).ready(function() {
updateSubModulesStatus();
$('#refreshSubmoduleStatus').click(function() { updateSubModulesStatus(); });
$('#updateAllJson').click(function() { updateAllJson(); });
});
$('#refreshSubmoduleStatus').click(function() { updateSubModulesStatus(); });
function updateSubModulesStatus(message) {
function updateSubModulesStatus(message, job_sent, sync_result) {
job_sent = job_sent === undefined ? false : job_sent;
sync_result = sync_result === undefined ? '' : sync_result;
$('#divSubmoduleVersions').empty().append('<it class="fa fa-spin fa-spinner" style="font-size: large; left: 50%; top: 50%;"></it>');
$.get('<?php echo $baseurl . '/servers/getSubmodulesStatus/'; ?>', function(html){
$('#divSubmoduleVersions').html(html);
if (message !== undefined) {
$('#submoduleGitResultDiv').show();
$('#submoduleGitResult').text(message);
var $clone = $('#submoduleGitResultDiv').clone();
$clone.find('strong').text('Synchronization result:');
if (job_sent) {
$clone.find('#submoduleGitResult')
.html('> Synchronizing DB with <a href="/jobs/index/" target="_blank">workers</a>...');
} else {
$clone.find('#submoduleGitResult')
.text(sync_result);
}
$clone.appendTo($('#submoduleGitResultDiv').parent());
}
});
}
function updateAllJson() {
$.ajax({
url: '<?php echo $baseurl . '/servers/updateJSON/'; ?>',
type: "get",
beforeSend: function() {
$('#submoduleGitResultDiv').show();
$('#submoduleGitResult').append('<it class="fa fa-spin fa-spinner" style="font-size: large; left: 50%; top: 50%;"></it>');
},
success: function(data, statusText, xhr) {
Object.keys(data).forEach(function(k) {
var val = data[k];
data[k] = val ? 'Updated' : 'Update failed';
});
$('#submoduleGitResult').html(syntaxHighlightJson(data));
},
complete: function() {
$('#submoduleGitResult').find('fa-spinner').remove();
}
});
}

2
app/View/Elements/healthElements/settings_row.ctp
View File

@ -22,6 +22,8 @@
'setting' => array(
'html' => h($setting['setting']),
'class' => 'short live_filter_target',
'ondblclick' => 'serverSettingsActivateField',
'ondblclickParams' => array(h($setting['setting']), h($k))
),
'value_passive' => array(
'html' => nl2br(h($setting['value'])),

20
app/View/Elements/histogram.ctp
View File

@ -9,19 +9,21 @@
<?php
$cnt = 0;
foreach ($typeDb as $type => $colour):
if (isset($data[0]['data'][$type])):
?>
<div class="attributehistogram-legend-line">
<div class="attributehistogram-legend-box" style="display: block;float: left;margin: 4px 6px 0 0;background-color:<?php echo $colour; ?>">&nbsp;</div>
<div style="display: inline-block;cursor: pointer;<?php if (in_array($type, $selectedTypes)) echo 'font-weight:bold';?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle histogram');?>" tite="<?php echo __('Toggle histogram');?>" onClick='toggleHistogramType("<?php echo h($type); ?>", [<?php foreach ($selectedTypes as $t) echo '"' . $t . '", ' ?>]);'><?php echo h($type);?></div>
</div>
<div class="attributehistogram-legend-line">
<div class="attributehistogram-legend-box" style="display: block;float: left;margin: 4px 6px 0 0;background-color:<?php echo $colour; ?>">&nbsp;</div>
<div style="display: inline-block;cursor: pointer;<?php if (in_array($type, $selectedTypes)) echo 'font-weight:bold';?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle histogram');?>" tite="<?php echo __('Toggle histogram');?>" onClick='toggleHistogramType("<?php echo h($type); ?>", [<?php foreach ($selectedTypes as $t) echo '"' . $t . '", ' ?>]);'><?php echo h($type);?></div>
</div>
<?php
if ($cnt % 12 == 11):
$cnt++;
endif;
if ($cnt % 12 == 0):
?>
</div>
<div class="attributehistogram-legend-line col">
</div>
<div class="attributehistogram-legend-line col">
<?php
endif;
$cnt++;
endif;
endforeach;
?>
</div>

9
app/View/Jobs/index.ctp
View File

@ -88,16 +88,9 @@
)
)
);
echo $this->element('/genericElements/ListTopBar/scaffold', array('data' => $data));
?>
<div id="attributeList" class="attributeListContainer">
<div class="tabMenu tabMenuFiltersBlock noPrint" style="padding-right:0px !important;">
<span id="filter_header" class="attribute_filter_header"><?php echo __('Filters');?>: </span>
<div id="filter_all" title="<?php echo __('Show all queues');?>" role="button" tabindex="0" aria-label="<?php echo __('Show all queues');?>" class="attribute_filter_text<?php if (!$queue) echo '_active';?>" onClick="window.location='/jobs/index';"><?php echo __('All');?></div>
<div id="filter_default" title="<?php echo __('Show default queue');?>" role="button" tabindex="0" aria-label="<?php echo __('Show default queue');?>" class="attribute_filter_text<?php if ($queue === 'default') echo '_active';?>" onClick="window.location='/jobs/index/default';"><?php echo __('Default');?></div>
<div id="filter_email" title="<?php echo __('Show email queue');?>" role="button" tabindex="0" aria-label="<?php echo __('Show email queue');?>" class="attribute_filter_text<?php if ($queue === 'email') echo '_active';?>" onClick="window.location='/jobs/index/email';"><?php echo __('Email');?></div>
<div id="filter_cache" title="<?php echo __('Show cache queue');?>" role="button" tabindex="0" aria-label="<?php echo __('Show cache queue');?>" class="attribute_filter_text<?php if ($queue === 'cache') echo '_active';?>" onClick="window.location='/jobs/index/cache';"><?php echo __('Cache');?></div>
</div>
<?php echo $this->element('/genericElements/ListTopBar/scaffold', array('data' => $data)); ?>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('id');?></th>

1
app/View/ObjectReferences/ajax/add.ctp
View File

@ -109,6 +109,7 @@
}
$options = array(
'functionName' => 'changeObjectReferenceSelectOption',
'select_threshold' => 0,
'chosen_options' => array('width' => '334px'),
'select_options' => array('data-targetselect' => 'targetSelect')
);

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit ac6276a906d0658ff3e021208db7f0d25316c65d
Subproject commit e0a5224bbb37608bd3f20c6ee992254f02b2c794

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit 7697111df5285a537a217bcc205d614355dc2127
Subproject commit 4092752c2ef60eba1df4a8bfdd55dbc68db10b73

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 3ededf3ddf92573e1037305859857418f73fdf25
Subproject commit 945b151b0b6258458fc0a8c3eb42f7401a30e5e8

4638
app/webroot/js/event-graph.js
View File

File diff suppressed because it is too large Load Diff

217
app/webroot/js/misp.js
View File

@ -1912,16 +1912,6 @@ function quickFilterRemoteEvents(passedArgs, id) {
window.location.href=url;
}
$('#quickFilterField').bind("enterKey",function(e){
$('#quickFilterButton').trigger("click");
});
$('#quickFilterField').keyup(function(e){
if(e.keyCode == 13)
{
$('#quickFilterButton').trigger("click");
}
});
function remoteIndexApplyFilters() {
var url = actionUrl + '/' + $("#EventFilter").val();
window.location.href = url;
@ -3311,33 +3301,6 @@ function flashErrorPopover() {
$("#gray_out").fadeIn();
}
$(".eventViewAttributeHover").mouseenter(function() {
$('#' + currentPopover).popover('destroy');
var type = $(this).attr('data-object-type');
var id = $(this).attr('data-object-id');
if (type + "_" + id in ajaxResults["hover"]) {
var element = $('#' + type + '_' + id + '_container');
element.popover({
title: attributeHoverTitle(id, type),
content: ajaxResults["hover"][type + "_" + id],
placement: attributeHoverPlacement(element),
html: true,
trigger: 'manual',
container: 'body'
}).popover('show');
currentPopover = type + '_' + id + '_container';
} else {
timer = setTimeout(function () {
runHoverLookup(type, id)
},
500
);
}
}).mouseout(function() {
clearTimeout(timer);
});
function attributeHoverTitle(id, type) {
return `<span>Lookup results:</span>
<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup"
@ -3378,15 +3341,6 @@ $('body').on('click', function (e) {
});
});
$(".queryPopover").click(function() {
url = $(this).data('url');
id = $(this).data('id');
$.get(url + '/' + id, function(data) {
$('#popover_form').html(data);
openPopup('#popover_form');
});
});
function serverOwnerOrganisationChange(host_org_id) {
if ($('#ServerOrganisationType').val() == "0" && $('#ServerLocal').val() == host_org_id) {
$('#InternalDiv').show();
@ -3488,26 +3442,6 @@ function feedFormUpdate() {
}
}
$('.servers_default_role_checkbox').click(function() {
var id = $(this).data("id");
var state = $(this).is(":checked");
$(".servers_default_role_checkbox").not(this).attr('checked', false);
$.ajax({
beforeSend: function (XMLHttpRequest) {
$(".loading").show();
},
success:function (data, textStatus) {
handleGenericAjaxResponse(data);
},
complete:function() {
$(".loading").hide();
},
type:"get",
cache: false,
url: '/admin/roles/set_default/' + (state ? id : ""),
});
});
function setContextFields() {
if (showContext) {
$('.context').show();
@ -3766,16 +3700,6 @@ function checkRolePerms() {
}
}
// clicking on an element with this class will select all of its contents in a
// single click
$('.quickSelect').click(function() {
var range = document.createRange();
var selection = window.getSelection();
range.selectNodeContents(this);
selection.removeAllRanges();
selection.addRange(range);
});
function updateMISP() {
$.get( "/servers/update", function(data) {
$("#confirmation_box").html(data);
@ -3822,7 +3746,9 @@ function submitSubmoduleUpdate(clicked) {
data: formData,
success:function (data, textStatus) {
if (data.status) {
updateSubModulesStatus(data.output);
var job_sent = data.job_sent !== undefined ? data.job_sent : false;
var sync_result = data.sync_result !== undefined ? data.sync_result : '';
updateSubModulesStatus(data.output, job_sent, sync_result);
} else {
showMessage('error', 'Something went wrong');
$('#submoduleGitResultDiv').show();
@ -3847,24 +3773,6 @@ function submitSubmoduleUpdate(clicked) {
});
}
$(".cortex-json").click(function() {
var cortex_data = $(this).data('cortex-json');
cortex_data = htmlEncode(JSON.stringify(cortex_data, null, 2));
var popupHtml = '<pre class="simplepre">' + cortex_data + '</pre>';
popupHtml += '<div class="close-icon useCursorPointer" onClick="closeScreenshot();"></div>';
$('#screenshot_box').html(popupHtml);
$('#screenshot_box').show();
$('#screenshot_box').css({'padding': '5px'});
left = ($(window).width() / 2) - ($('#screenshot_box').width() / 2);
if (($('#screenshot_box').height() + 250) > $(window).height()) {
$('#screenshot_box').height($(window).height() - 250);
$('#screenshot_box').css("overflow-y", "scroll");
$('#screenshot_box').css("overflow-x", "hidden");
}
$('#screenshot_box').css({'left': left + 'px'});
$("#gray_out").fadeIn();
});
// Show $(id) if the enable parameter evaluates to true. Hide it otherwise
function checkAndEnable(id, enable) {
if (enable) {
@ -4005,22 +3913,6 @@ function previewEventBasedOnUuids() {
}
}
$('.add_object_attribute_row').click(function() {
var template_id = $(this).data('template-id');
var object_relation = $(this).data('object-relation');
var k = $('#last-row').data('last-row');
var k = k+1;
$('#last-row').data('last-row', k);
url = "/objects/get_row/" + template_id + "/" + object_relation + "/" + k;
$.get(url, function(data) {
$('#row_' + object_relation + '_expand').before($(data).fadeIn()).html();
});
});
$('.quickToggleCheckbox').toggle(function() {
var url = $(this).data('checkbox-url');
});
function checkNoticeList(type) {
var fields_to_check = {
"attribute": ["category", "type"]
@ -4049,6 +3941,108 @@ function checkNoticeList(type) {
}
$(document).ready(function() {
$('#quickFilterField').bind("enterKey",function(e){
$('#quickFilterButton').trigger("click");
});
$('#quickFilterField').keyup(function(e){
if(e.keyCode == 13)
{
$('#quickFilterButton').trigger("click");
}
});
$(".eventViewAttributeHover").mouseenter(function() {
$('#' + currentPopover).popover('destroy');
var type = $(this).attr('data-object-type');
var id = $(this).attr('data-object-id');
if (type + "_" + id in ajaxResults["hover"]) {
var element = $('#' + type + '_' + id + '_container');
element.popover({
title: attributeHoverTitle(id, type),
content: ajaxResults["hover"][type + "_" + id],
placement: attributeHoverPlacement(element),
html: true,
trigger: 'manual',
container: 'body'
}).popover('show');
currentPopover = type + '_' + id + '_container';
} else {
timer = setTimeout(function () {
runHoverLookup(type, id)
},
500
);
}
}).mouseout(function() {
clearTimeout(timer);
});
$(".queryPopover").click(function() {
url = $(this).data('url');
id = $(this).data('id');
$.get(url + '/' + id, function(data) {
$('#popover_form').html(data);
openPopup('#popover_form');
});
});
$('.servers_default_role_checkbox').click(function() {
var id = $(this).data("id");
var state = $(this).is(":checked");
$(".servers_default_role_checkbox").not(this).attr('checked', false);
$.ajax({
beforeSend: function (XMLHttpRequest) {
$(".loading").show();
},
success:function (data, textStatus) {
handleGenericAjaxResponse(data);
},
complete:function() {
$(".loading").hide();
},
type:"get",
cache: false,
url: '/admin/roles/set_default/' + (state ? id : ""),
});
});
// clicking on an element with this class will select all of its contents in a
// single click
$('.quickSelect').click(function() {
var range = document.createRange();
var selection = window.getSelection();
range.selectNodeContents(this);
selection.removeAllRanges();
selection.addRange(range);
});
$(".cortex-json").click(function() {
var cortex_data = $(this).data('cortex-json');
cortex_data = htmlEncode(JSON.stringify(cortex_data, null, 2));
var popupHtml = '<pre class="simplepre">' + cortex_data + '</pre>';
popupHtml += '<div class="close-icon useCursorPointer" onClick="closeScreenshot();"></div>';
$('#screenshot_box').html(popupHtml);
$('#screenshot_box').show();
$('#screenshot_box').css({'padding': '5px'});
left = ($(window).width() / 2) - ($('#screenshot_box').width() / 2);
if (($('#screenshot_box').height() + 250) > $(window).height()) {
$('#screenshot_box').height($(window).height() - 250);
$('#screenshot_box').css("overflow-y", "scroll");
$('#screenshot_box').css("overflow-x", "hidden");
}
$('#screenshot_box').css({'left': left + 'px'});
$("#gray_out").fadeIn();
});
$('.add_object_attribute_row').click(function() {
var template_id = $(this).data('template-id');
var object_relation = $(this).data('object-relation');
var k = $('#last-row').data('last-row');
var k = k+1;
$('#last-row').data('last-row', k);
url = "/objects/get_row/" + template_id + "/" + object_relation + "/" + k;
$.get(url, function(data) {
$('#row_' + object_relation + '_expand').before($(data).fadeIn()).html();
});
});
$('.quickToggleCheckbox').toggle(function() {
var url = $(this).data('checkbox-url');
});
$(".correlation-expand-button").on("click", function() {
$(this).parent().children(".correlation-expanded-area").show();
$(this).parent().children(".correlation-collapse-button").show();
@ -4105,7 +4099,6 @@ function insertHTMLRestResponse() {
function insertJSONRestResponse() {
$('#rest-response-container').append('<p id="json-response-container" style="border: 1px solid blue; padding:5px;" />');
var parsedJson = syntaxHighlightJson($('#rest-response-hidden-container').text());
console.log(parsedJson);
$('#json-response-container').html(parsedJson);
}

65
docs/INSTALL.rhel7.md
View File

@ -116,24 +116,23 @@ systemctl enable --now rh-redis32-redis.service
scl enable rh-mariadb102 'mysql_secure_installation'
```
## 2.09/ Update the PHP extension repository and install required package
```bash
scl enable rh-php72 rh-redis32 bash
pear channel-update pear.php.net
pear install Crypt_GPG
```
## 2.10/ Install haveged and enable to start on boot to provide entropy for GPG
## 2.09/ Optional: install haveged and enable to start on boot to provide entropy for GPG
```bash
yum install haveged
systemctl enable --now haveged
```
Only do this if you're not running rngd to provide randomness and your kernel randomness is not sufficient.
## 2.11/ Install Python 3.6 from SCL
## 2.10/ Install Python 3.6 from SCL
```bash
yum install rh-python36
```
## 2.11/ Install Git 2.18 from SCL
```bash
yum install rh-git218
```
# 3/ MISP Download
## 3.01/ Download MISP code using git in /var/www/ directory
```bash
@ -195,10 +194,11 @@ systemctl restart rh-php72-php-fpm.service
## 3.06/ Enable dependencies detection in the diagnostics page
Add the following content to `/etc/opt/rh/rh-php72/php-fpm.d/www.conf` :
```
env[PATH] =/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin
env[PATH]=/opt/rh/rh-git218/root/usr/bin:/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-python36/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin
env[LD_LIBRARY_PATH]=/opt/rh/httpd24/root/usr/lib64/
```
Then run `systemctl restart rh-php72-php-fpm.service`.
This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings. The LD_LIBRARY_PATH setting is needed for rh-git218 to work, one might think to install httpd24 and not just httpd ...
# 4/ CakePHP
## 4.01/ Install CakeResque along with its dependencies if you intend to use the built in background jobs
@ -254,8 +254,6 @@ chown -R apache:apache /var/www/MISP/app/webroot/img/orgs
chown -R apache:apache /var/www/MISP/app/webroot/img/custom
```
# 6/ Create database and user
## 6.01/ Set database to listen on localhost only
```bash
@ -290,14 +288,10 @@ cp /var/www/MISP/INSTALL/apache.misp.centos7 /etc/httpd/conf.d/misp.conf
## 7.02/ Since SELinux is enabled, we need to allow httpd to write to certain directories
```bash
chcon -t httpd_sys_rw_content_t /var/www/MISP/app/files
chcon -t httpd_sys_rw_content_t /var/www/MISP/app/files/terms
chcon -t httpd_sys_rw_content_t /var/www/MISP/app/files/scripts/tmp
chcon -t httpd_sys_rw_content_t /var/www/MISP/app/Plugin/CakeResque/tmp
chcon -R -t httpd_sys_rw_content_t /var/www/MISP/app/tmp
chcon -R -t httpd_sys_rw_content_t /var/www/MISP/app/webroot/img/orgs
chcon -R -t httpd_sys_rw_content_t /var/www/MISP/app/webroot/img/custom
semanage fcontext -a -t httpd_sys_rw_content_t "/var/www/MISP(/.*)?"
restorecon -R /var/www/MISP/
```
We're providing write access to the whole MISP tree, otherwise updates via the web interface won't work.
## 7.03/ Allow httpd to connect to the redis server and php-fpm over tcp/ip
```bash
@ -329,13 +323,7 @@ cp INSTALL/misp.logrotate /etc/logrotate.d/misp
chmod 0640 /etc/logrotate.d/misp
```
## 8.02/ Allow logrotate to work under SELinux and modify the log files
```bash
semanage fcontext -a -t httpd_log_t "/var/www/MISP/app/tmp/logs(/.*)?"
chcon -R -t httpd_log_t /var/www/MISP/app/tmp/logs
```
## 8.03/ Allow logrotate to read /var/www
## 8.02/ Allow logrotate to read /var/www
```bash
checkmodule -M -m -o /tmp/misplogrotate.mod INSTALL/misplogrotate.te
semodule_package -o /tmp/misplogrotate.pp -m /tmp/misplogrotate.mod
@ -381,17 +369,13 @@ cp -a config.default.php config.php
```
## 9.03/ If you want to be able to change configuration parameters from the webinterface:
```
chown apache:apache /var/www/MISP/app/Config/config.php
chcon -t httpd_sys_rw_content_t /var/www/MISP/app/Config/config.php
```
Handled by 7.02
## 9.04/ Generate an encryption key
```bash
gpg --gen-key
mv ~/.gnupg /var/www/MISP/
chown -R apache:apache /var/www/MISP/.gnupg
chcon -R -t httpd_sys_rw_content_t /var/www/MISP/.gnupg
restorecon -R /var/www/MISP
```
!!! note
@ -443,16 +427,7 @@ systemctl enable --now misp-workers.service
{!generic/recommended.actions.md!}
# 10/ Post Install
## 10.01/ Allow apache to write to /var/www/MISP/app/tmp/logs
If the result from the diagnostic page is that the directory is not writable, try the following.
```
chcon -R -t httpd_sys_rw_content_t /var/www/MISP/app/tmp/logs/
```
!!! note
This may mean that logrotate cannot access the logs directory, will require further investigation
## 10.02/ Change php.ini settings to suggested limits from diagnostic page.
## 10.01/ Change php.ini settings to suggested limits from diagnostic page.
```bash
# Edit /etc/opt/rh/rh-php72/php.ini and set the following settings
max_execution_time = 300
@ -461,12 +436,12 @@ upload_max_filesize = 50M
post_max_size = 50M
```
## 10.03/ Restart rh-php72 for settings to take effect
## 10.02/ Restart rh-php72 for settings to take effect
```bash
systemctl restart rh-php72-php-fpm
```
## 10.04/ Install pydeep and pymisp
## 10.03/ Install pydeep and pymisp
```bash
scl enable rh-python36 'python3 -m pip install pymisp git+https://github.com/kbandla/pydeep.git'
```