MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of https://github.com/MISP/MISP into rework_modules

pull/4584/head
chrisr3d 2019-04-12 23:53:00 +02:00
parent 8a9612b376 c527077b1c
commit fdc726e609
17 changed files with 2513 additions and 2421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PyMISP

@ -1 +1 @@
Subproject commit a68bd80ab9dceaee9674bd9a2b0bffc4f387fcdc
Subproject commit e8334be9caaddab229d850c92cb029812222dd82

59
app/Console/worker/start.sh
View File

@ -1,57 +1,18 @@
#!/usr/bin/env bash
# TODO: Put some logic inside if many worker PIDs are detected
# Extract base directory where this script is and cd into it
cd "${0%/*}"
# Set to the current webroot owner
WWW_USER=$(ls -l ../cake |awk {'print $3'}|tail -1)
# In most cases the owner of the cake script is also the user as which it should be executed.
if [[ "$USER" != "$WWW_USER" ]]; then
echo "You run this script as $USER and the owner of the cake command is $WWW_USER. This might be an issue."
fi
# Check if run as root
if [[ "$EUID" -eq "0" ]]; then
if [ "$EUID" -eq 0 ]; then
echo "Please DO NOT run the worker script as root"
exit 1
fi
# Check if jq is present and enable advanced checks
if [[ "$(jq -V > /dev/null 2> /dev/null; echo $?)" != 0 ]]; then
echo "jq is not installed, disabling advanced checks."
ADVANCED="0"
else
ADVANCED="1"
fi
# Extract base directory where this script is and cd into it
cd "${0%/*}"
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque startscheduler --interval 5
if [[ "$ADVANCED" == "1" ]]; then
for worker in `echo cache default email prio scheduler`; do
workerStatus=$(../cake Admin getWorkers |tail -n +7 |jq -r ".$worker" |jq -r '.ok')
PIDcount=$(../cake admin getWorkers |tail -n +7 |jq -r ".$worker.workers" |grep pid | wc -l)
echo -n "$worker has $PIDcount PID(s)"
if [[ "$workerStatus" != "true" ]]; then
echo ", trying to restart."
if [[ "$worker" != "scheduler" ]]; then
../cake CakeResque.CakeResque start --interval 5 --queue $worker
else
../cake CakeResque.CakeResque startscheduler --interval 5
fi
else
echo ", up and running."
fi
done
exit 0
else
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque startscheduler --interval 5
exit 0
fi
exit 0

57
app/Console/worker/start_dynamic.sh Normal file
View File

@ -0,0 +1,57 @@
#!/usr/bin/env bash
# TODO: Put some logic inside if many worker PIDs are detected
# Extract base directory where this script is and cd into it
cd "${0%/*}"
# Set to the current webroot owner
WWW_USER=$(ls -l ../cake |awk {'print $3'}|tail -1)
# In most cases the owner of the cake script is also the user as which it should be executed.
if [[ "$USER" != "$WWW_USER" ]]; then
echo "You run this script as $USER and the owner of the cake command is $WWW_USER. This might be an issue."
fi
# Check if run as root
if [[ "$EUID" -eq "0" ]]; then
echo "Please DO NOT run the worker script as root"
exit 1
fi
# Check if jq is present and enable advanced checks
if [[ "$(jq -V > /dev/null 2> /dev/null; echo $?)" != 0 ]]; then
echo "jq is not installed, disabling advanced checks."
ADVANCED="0"
else
ADVANCED="1"
fi
if [[ "$ADVANCED" == "1" ]]; then
for worker in `echo cache default email prio scheduler`; do
workerStatus=$(../cake Admin getWorkers |tail -n +7 |jq -r ".$worker" |jq -r '.ok')
PIDcount=$(../cake admin getWorkers |tail -n +7 |jq -r ".$worker.workers" |grep pid | wc -l)
echo -n "$worker has $PIDcount PID(s)"
if [[ "$workerStatus" != "true" ]]; then
echo ", trying to restart."
if [[ "$worker" != "scheduler" ]]; then
../cake CakeResque.CakeResque start --interval 5 --queue $worker
else
../cake CakeResque.CakeResque startscheduler --interval 5
fi
else
echo ", up and running."
fi
done
exit 0
else
../cake CakeResque.CakeResque stop --all
../cake CakeResque.CakeResque start --interval 5 --queue default
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque startscheduler --interval 5
exit 0
fi

52
app/Controller/AttributesController.php
View File

@ -1197,7 +1197,7 @@ class AttributesController extends AppController
}
}
public function viewPicture($id, $thumbnail=false, $width=200, $height=200)
public function viewPicture($id, $thumbnail=false)
{
if (Validation::uuid($id)) {
$temp = $this->Attribute->find('first', array(
@ -1221,11 +1221,15 @@ class AttributesController extends AppController
'Attribute.id' => $id,
'Attribute.type' => 'attachment'
),
'withAttachments' => true,
'includeAllTags' => false,
'includeAttributeUuid' => true,
'flatten' => true
);
if ($this->_isRest()) {
$conditions['withAttachments'] = true;
}
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $conditions);
if (empty($attribute)) {
throw new MethodNotAllowedException('Invalid attribute');
@ -1235,49 +1239,11 @@ class AttributesController extends AppController
if ($this->_isRest()) {
return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type());
} else {
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200;
$image_data = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height);
$extension = explode('.', $attribute['Attribute']['value']);
$extension = end($extension);
if (extension_loaded('gd')) {
$image = ImageCreateFromString(base64_decode($attribute['Attribute']['data']));
if (!$thumbnail) {
ob_start ();
switch ($extension) {
case 'gif':
imagegif($image);
break;
case 'jpg':
case 'jpeg':
imagejpeg($image);
break;
case 'png':
imagepng($image);
break;
default:
break;
}
$image_data = $extension != 'gif' ? ob_get_contents() : base64_decode($attribute['Attribute']['data']);
ob_end_clean ();
imagedestroy($image);
} else { // thumbnail requested, resample picture with desired dimension
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 150;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 150;
if ($extension == 'gif') {
$image_data = base64_decode($attribute['Attribute']['data']);
} else {
$extension = 'jpg';
$imageTC = ImageCreateTrueColor($width, $height);
ImageCopyResampled($imageTC, $image, 0, 0, 0, 0, $width, $height, ImageSX($image), ImageSY($image));
ob_start ();
imagejpeg ($imageTC);
$image_data = ob_get_contents();
ob_end_clean ();
imagedestroy($image);
imagedestroy($imageTC);
}
}
} else {
$image_data = base64_decode($attribute['Attribute']['data']);
}
$this->response->type(strtolower(h($extension)));
$this->response->body($image_data);
$this->autoRender = false;

2
app/Controller/Component/RestResponseComponent.php
View File

@ -1552,7 +1552,7 @@ class RestResponseComponent extends Component
$field['values'] = $tags;
}
private function __overwriteNationality($scope, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Organisation")->countries);
$field['values'] = ClassRegistry::init("Organisation")->countries;
}
private function __overwriteAction($scope, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Log")->actionDefinitions);

1
app/Controller/TaxonomiesController.php
View File

@ -60,6 +60,7 @@ class TaxonomiesController extends AppController
$this->loadModel('AttributeTag');
foreach ($taxonomy['entries'] as $key => $value) {
$count = 0;
$count_a = 0;
if (!empty($value['existing_tag'])) {
foreach ($value['existing_tag'] as $et) {
$count = $this->EventTag->find('count', array(

4
app/Lib/Export/StixExport.php
View File

@ -58,7 +58,7 @@ class StixExport
$randomFileName = $this->generateRandomFileName();
$this->__tmp_dir = $this->__scripts_dir . 'tmp/';
$this->__framing = json_decode(shell_exec($framing_cmd), true);
$this->__stix_file = new File($this->__tmp_dir . $randomFileName . '.stix');
$this->__stix_file = new File($this->__tmp_dir . $randomFileName . '.' . $this->__return_type);
unset($randomFileName);
$this->__stix_file->write($this->__framing['header']);
$this->__initialize_misp_file();
@ -93,7 +93,7 @@ class StixExport
$this->__stix_file->close();
$this->__stix_file->delete();
$sep_len = strlen($this->__framing['separator']);
$stix_event = substr($stix_event, 0, -$sep_len) . $this->__framing['footer'];
$stix_event = (empty($this->__filenames) ? $stix_event : substr($stix_event, 0, -$sep_len)) . $this->__framing['footer'];
return $stix_event;
}

86
app/Model/Attribute.php
View File

@ -1589,7 +1589,7 @@ class Attribute extends AppModel
}
}
public function base64EncodeAttachment($attribute)
public function getAttachment($attribute, $path_suffix='')
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
@ -1600,21 +1600,20 @@ class Attribute extends AppModel
// S3 - we have to first get the object then we can encode it
$s3 = $this->getS3Client();
// This will return the content of the object
$content = $s3->download($attribute['event_id'] . DS . $attribute['id']);
$content = $s3->download($attribute['event_id'] . DS . $attribute['id'] . $path_suffix);
} else {
// Standard filesystem
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
$file = new File($filepath);
if (!$file->readable()) {
return '';
}
$content = $file->read();
}
return base64_encode($content);
return $content;
}
public function saveBase64EncodedAttachment($attribute)
public function saveAttachment($attribute, $path_suffix='')
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
@ -1626,17 +1625,17 @@ class Attribute extends AppModel
// We don't need your fancy directory structures and
// PEE AICH PEE meddling
$s3 = $this->getS3Client();
$data = base64_decode($attribute['data']);
$key = $attribute['event_id'] . DS . $attribute['id'];
$data = $attribute['data'];
$key = $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
$s3->upload($key, $data);
return true;
} else {
// Plebian filesystem operations
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$destpath = $rootDir . DS . $attribute['id'] . $path_suffix;
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
$decodedData = $attribute['data']; // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
@ -1646,6 +1645,73 @@ class Attribute extends AppModel
}
}
public function base64EncodeAttachment($attribute)
{
return base64_encode($this->getAttachment($attribute));
}
public function saveBase64EncodedAttachment($attribute)
{
$attribute['data'] = base64_decode($attribute['data']);
return $this->saveAttachment($attribute);
}
public function getPictureData($attribute, $thumbnail=false, $width=200, $height=200)
{
$extension = explode('.', $attribute['Attribute']['value']);
$extension = end($extension);
if (extension_loaded('gd')) {
if (!$thumbnail) {
$data = $this->getAttachment($attribute['Attribute']);
$image = ImageCreateFromString($data);
ob_start ();
switch ($extension) {
case 'gif':
// php-gd doesn't support animated gif. Skipping...
break;
case 'jpg':
case 'jpeg':
imagejpeg($image);
break;
case 'png':
imagepng($image);
break;
default:
break;
}
$image_data = $extension != 'gif' ? ob_get_contents() : $data;
ob_end_clean ();
} else { // thumbnail requested, resample picture with desired dimension and save result
$thumbnail_exists = $this->getAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
if ($width == 200 && $height == 200 && $thumbnail_exists !== '') { // check if thumbnail already exists
$image_data = $thumbnail_exists;
} else {
$data = $this->getAttachment($attribute['Attribute']);
if ($extension == 'gif') {
$image_data = $data;
} else {
$image = ImageCreateFromString($data);
$extension = 'jpg';
$imageTC = ImageCreateTrueColor($width, $height);
ImageCopyResampled($imageTC, $image, 0, 0, 0, 0, $width, $height, ImageSX($image), ImageSY($image));
ob_start ();
imagejpeg ($imageTC);
$image_data = ob_get_contents();
ob_end_clean ();
imagedestroy($image);
imagedestroy($imageTC);
}
// save thumbnail for later reuse
$attribute['Attribute']['data'] = $image_data;
$this->saveAttachment($attribute['Attribute'], '_thumbnail');
}
}
} else {
$image_data = $this->getAttachment($attribute['Attribute']);
}
return $image_data;
}
public function __beforeSaveCorrelation($a)
{
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id

1
app/Model/Event.php
View File

@ -1647,7 +1647,6 @@ class Event extends AppModel
'last',
'to_ids',
'includeAllTags',
'withAttachments',
'includeAttachments',
'event_uuid',
'distribution',

2
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -594,7 +594,7 @@
));
echo $this->element('/genericElements/SideMenu/side_menu_post_link', array(
'event_id' => 'deleteUser',
'url' => '/admin/delete/' . h($id),
'url' => '/admin/users/delete/' . h($id),
'text' => __('Delete User'),
'message' => __('Are you sure you want to delete # %s? It is highly recommended to never delete users but to disable them instead.', h($id))
));

2
app/View/Elements/healthElements/settings_row.ctp
View File

@ -22,6 +22,8 @@
'setting' => array(
'html' => h($setting['setting']),
'class' => 'short live_filter_target',
'ondblclick' => 'serverSettingsActivateField',
'ondblclickParams' => array(h($setting['setting']), h($k))
),
'value_passive' => array(
'html' => nl2br(h($setting['value'])),

20
app/View/Elements/histogram.ctp
View File

@ -9,19 +9,21 @@
<?php
$cnt = 0;
foreach ($typeDb as $type => $colour):
if (isset($data[0]['data'][$type])):
?>
<div class="attributehistogram-legend-line">
<div class="attributehistogram-legend-box" style="display: block;float: left;margin: 4px 6px 0 0;background-color:<?php echo $colour; ?>">&nbsp;</div>
<div style="display: inline-block;cursor: pointer;<?php if (in_array($type, $selectedTypes)) echo 'font-weight:bold';?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle histogram');?>" tite="<?php echo __('Toggle histogram');?>" onClick='toggleHistogramType("<?php echo h($type); ?>", [<?php foreach ($selectedTypes as $t) echo '"' . $t . '", ' ?>]);'><?php echo h($type);?></div>
</div>
<div class="attributehistogram-legend-line">
<div class="attributehistogram-legend-box" style="display: block;float: left;margin: 4px 6px 0 0;background-color:<?php echo $colour; ?>">&nbsp;</div>
<div style="display: inline-block;cursor: pointer;<?php if (in_array($type, $selectedTypes)) echo 'font-weight:bold';?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle histogram');?>" tite="<?php echo __('Toggle histogram');?>" onClick='toggleHistogramType("<?php echo h($type); ?>", [<?php foreach ($selectedTypes as $t) echo '"' . $t . '", ' ?>]);'><?php echo h($type);?></div>
</div>
<?php
if ($cnt % 12 == 11):
$cnt++;
endif;
if ($cnt % 12 == 0):
?>
</div>
<div class="attributehistogram-legend-line col">
</div>
<div class="attributehistogram-legend-line col">
<?php
endif;
$cnt++;
endif;
endforeach;
?>
</div>

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit ac6276a906d0658ff3e021208db7f0d25316c65d
Subproject commit e0a5224bbb37608bd3f20c6ee992254f02b2c794

2
app/files/scripts/stix2/misp2stix2.py
View File

@ -70,7 +70,7 @@ class StixBuilder():
'created_by_ref': self.identity_id, 'created': self.misp_event['date'],
'published': self.get_datetime_from_timestamp(self.misp_event['publish_timestamp']),
'interoperability': True}
labels = _MISP_event_tags
labels = [tag for tag in _MISP_event_tags]
if self.misp_event.get('Tag'):
markings = []
for tag in self.misp_event['Tag']:

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit 7697111df5285a537a217bcc205d614355dc2127
Subproject commit 4092752c2ef60eba1df4a8bfdd55dbc68db10b73

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 3ededf3ddf92573e1037305859857418f73fdf25
Subproject commit 945b151b0b6258458fc0a8c3eb42f7401a30e5e8

4638
app/webroot/js/event-graph.js
View File

File diff suppressed because it is too large Load Diff