MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of https://github.com/MISP/MISP into misp-stix

pull/8045/head
chrisr3d 2021-12-13 04:11:47 +01:00
parent 8af7a00073 a546fc7919
commit fadfd10835
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
158 changed files with 9435 additions and 5672 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
.github/ISSUE_TEMPLATE/bug-form.yml vendored Normal file
View File

@ -0,0 +1,88 @@
name: Bug Report
description: Create a report to help us improve
title: "Bug: "
labels: ["bug", "potential-bug", "needs triage"]
body:
- type: markdown
attributes:
value: |
Please read the [FAQ](https://www.circl.lu/doc/misp/faq/) before opening an issue.
If you would like to report a bug, please fill the template bellow:
- type: textarea
id: expected-behavior
attributes:
label: Expected behavior
validations:
required: true
- type: textarea
id: actual-behavior
attributes:
label: Actual behavior
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to reproduce
description: Please be as thorough as possible.
validations:
required: true
- type: input
id: misp-version
attributes:
label: Version
description: What version of MISP are you running?
placeholder: "2.4.151"
validations:
required: true
- type: input
id: os
attributes:
label: Operating System
placeholder: "Ubuntu, RedHat, CentOS ..."
validations:
required: true
- type: input
id: os-version
attributes:
label: Operating System version
placeholder: "20.04"
validations:
required: true
- type: input
id: php-version
attributes:
label: PHP version
placeholder: "7.0, 7.1, 7.4 ..."
validations:
required: true
- type: input
id: browser
attributes:
label: Browser
description: Which browser are you seeing the problem on?
placeholder: "Firefox, Chrome, Safari ..."
- type: input
id: browser-version
attributes:
label: Browser version
- type: textarea
id: logs
attributes:
label: Relevant log output
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
- type: textarea
id: extra-attachments
attributes:
label: Extra attachments
description: Please add any other relevant attachments such as screenshots, log files, etc. here.
- type: checkboxes
id: terms
attributes:
label: Code of Conduct
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/MISP/MISP/blob/2.4/code_of_conduct.md)
options:
- label: I agree to follow this project's Code of Conduct
required: true

40
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@ -1,40 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
labels: bug, potential-bug, needs triage
---
# This template is meant for bug reports, if you have a feature request user the other template.
*If you would like to report a bug, please fill the template bellow*
### Work environment
| Questions | Answers
|---------------------------|--------------------
| Type of issue | Bug
| OS version (server) | Debian, Ubuntu, CentOS, RedHat, ...
| OS version (client) | MacOS, Win10, Ubuntu, ...
| PHP version | 5.4, 5.5, 5.6, 7.0, 7.1...
| MISP version / git hash | 2.4.XX, hash of the commit
| Browser | If applicable
### Expected behavior
### Actual behavior
### Steps to reproduce the behavior
### Logs, screenshots, configuration dump, ...

11
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@ -0,0 +1,11 @@
blank_issues_enabled: true
contact_links:
- name: Discussions
url: https://github.com/MISP/MISP/discussions
about: For more general questions or ongoing debates.
- name: FAQ
url: https://www.circl.lu/doc/misp/faq/
about: Please find our Frequently Asked Questions here.
- name: Community Support
url: https://gitter.im/MISP/Support
about: For Community Support reach out here.

37
.github/ISSUE_TEMPLATE/feature-request-form.yml vendored Normal file
View File

@ -0,0 +1,37 @@
name: Feature request
description: Suggest an idea for this project
title: "Feature Request: "
labels: ["feature request", "needs triage"]
body:
- type: textarea
id: motif
attributes:
label: Is your feature request related to a problem? Please describe.
placeholder: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
validations:
required: true
- type: textarea
id: solution
attributes:
label: Describe the solution you'd like
placeholder: A clear and concise description of what you want to happen.
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Describe alternatives you've considered
placeholder: A clear and concise description of any alternative solutions or features you've considered.
- type: textarea
id: context
attributes:
label: Additional context
placeholder: Add any other context or screenshots about the feature request here.
- type: checkboxes
id: terms
attributes:
label: Code of Conduct
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/MISP/MISP/blob/2.4/code_of_conduct.md)
options:
- label: I agree to follow this project's Code of Conduct
required: true

19
.github/ISSUE_TEMPLATE/feature_request.md vendored
View File

@ -1,19 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
labels: feature request, needs triage
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

85
.github/ISSUE_TEMPLATE/support-form.yml vendored Normal file
View File

@ -0,0 +1,85 @@
name: Support Request
description: Support requests for MISP
title: "Support: "
labels: ["support", "needs triage"]
body:
- type: markdown
attributes:
value: |
## Please consider the following notes
- Critical security bugs can be reported, preferably PGP encrypted, by mail to: info@circl.lu
- Bug reports and feature requests can be filed as an issue on github: https://github.com/MISP/MISP/issues
- For interactive support please join the Gitter chat on: https://gitter.im/MISP/MISP
The official documentation of MISP can be found here: https://www.circl.lu/doc/misp/
Please read the [FAQ](https://www.circl.lu/doc/misp/faq/) before opening an issue.
We also periodically do user/admin/developer trainings and have our training material online: https://www.circl.lu/services/misp-training-materials/
Nevertheless you can of course file a Support request as an issue. Please be as precise as possible and fill the template as detailed as possible too.
If you would like to ask for support, please fill the template bellow:
- type: textarea
id: support-questions
attributes:
label: Support Questions
validations:
required: true
- type: input
id: misp-version
attributes:
label: MISP version
description: What version of MISP are you running?
placeholder: "2.4.151"
validations:
required: true
- type: input
id: os
attributes:
label: Operating System
placeholder: "Ubuntu, RedHat, CentOS ..."
validations:
required: true
- type: input
id: os-version
attributes:
label: Operating System version
placeholder: "20.04"
validations:
required: true
- type: input
id: php-version
attributes:
label: PHP version
placeholder: "7.0, 7.1, 7.4 ..."
validations:
required: true
- type: input
id: browser
attributes:
label: Browser
description: Which browser are you seeing the problem on?
placeholder: "Firefox, Chrome, Safari ..."
- type: input
id: browser-version
attributes:
label: Browser version
- type: textarea
id: logs
attributes:
label: Relevant log output
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
- type: textarea
id: extra-attachments
attributes:
label: Extra attachments
description: Please add any other relevant attachments such as screenshots, log files, etc. here.
- type: checkboxes
id: terms
attributes:
label: Code of Conduct
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/MISP/MISP/blob/2.4/code_of_conduct.md)
options:
- label: I agree to follow this project's Code of Conduct
required: true

34
.github/ISSUE_TEMPLATE/support_request.md vendored
View File

@ -1,34 +0,0 @@
---
name: Support
about: Support requests for MISP
labels: support, needs triage
---
## Please consider the following notes
- Critical security bugs can be reported, preferably PGP encrypted, by mail to: info@circl.lu
- Bug reports and feature requests can be filed as an issue on github: https://github.com/MISP/MISP/issues
- For interactive support please join the Gitter chat on: https://gitter.im/MISP/MISP
The official documentation of MISP can be found here: https://www.circl.lu/doc/misp/
We also periodically do user/admin/developer trainings and have our training material online: https://www.circl.lu/services/misp-training-materials/
Nevertheless you can of course file a Support request as an issue. Please be as precise as possible and fill the template as detailed as possible too.
Please remove this text until the line below. Thanks a lot.
--------8<------
### Work environment
| Questions | Answers
|---------------------------|--------------------
| Type of issue | Support
| OS version (server) | Debian, Ubuntu, CentOS, RedHat, ...
| OS version (client) | MacOS, Win10, Ubuntu, ...
| PHP version | 5.4, 5.5, 5.6, 7.0, 7.1... (if relevant)
| MISP version / git hash | 2.4.XX, hash of the commit
| Browser | If applicable
### Support Questions
### Logs, screenshots, configuration dump, ...

25
.github/workflows/main.yml vendored
View File

@ -54,7 +54,7 @@ jobs:
uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php }}
extensions: mysql, mbstring, json, xml, opcache, readline, redis, gd
extensions: mysql, mbstring, json, xml, opcache, readline, redis, gd, apcu
- name: Initialize variables
run: |
@ -71,6 +71,8 @@ jobs:
LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
sudo apt-get -y install curl python3 python3-zmq python3-requests python3-pip python3-nose python3-redis python3-lxml apache2 libapache2-mod-php$php_version libfuzzy-dev
sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu packages
# hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel
sudo apt install --only-upgrade libpcre2-8-0
curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -
sudo chown $USER:www-data $HOME/.composer
pushd app
@ -126,9 +128,16 @@ jobs:
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 770 `pwd`/.gnupg
# Get authkey
sudo chown -R www-data:www-data `pwd`/.gnupg
sudo chmod -R 700 `pwd`/.gnupg
sudo usermod -a -G www-data $USER
sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
- name: DB Update
run: |
@ -185,7 +194,7 @@ jobs:
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -E su $USER -c 'app/Console/worker/start.sh'
sudo -u www-data 'app/Console/worker/start.sh'
- name: Python setup
run: |
@ -237,13 +246,6 @@ jobs:
run: |
export PATH=$HOME/.local/env:$PATH # enable poetry binary
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
pushd tests
./curl_tests_GH.sh $AUTH $HOST
popd
@ -255,6 +257,7 @@ jobs:
poetry add lxml
poetry run python ../tests/testlive_security.py -v
poetry run python ../tests/testlive_sync.py
poetry run python ../tests/testlive_comprehensive_local.py -v
poetry run python tests/test_mispevent.py
popd
cp PyMISP/tests/keys.py PyMISP/examples/events/

4
INSTALL/MYSQL.sql
View File

@ -548,7 +548,7 @@ CREATE TABLE IF NOT EXISTS `jobs` (
`message` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
`progress` int(11) NOT NULL DEFAULT 0,
`org_id` int(11) NOT NULL DEFAULT 0,
`process_id` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,
`process_id` varchar(36) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,
`date_created` datetime NOT NULL,
`date_modified` datetime NOT NULL,
PRIMARY KEY (`id`)
@ -690,6 +690,7 @@ CREATE TABLE IF NOT EXISTS `object_references` (
`comment` text COLLATE utf8_bin NOT NULL,
`deleted` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (id),
UNIQUE INDEX `uuid` (`uuid`),
INDEX `source_uuid` (`source_uuid`),
INDEX `referenced_uuid` (`referenced_uuid`),
INDEX `timestamp` (`timestamp`),
@ -1056,6 +1057,7 @@ CREATE TABLE IF NOT EXISTS `sharing_groups` (
INDEX `org_id` (`org_id`),
INDEX `sync_user_id` (`sync_user_id`),
UNIQUE INDEX `uuid` (`uuid`),
UNIQUE INDEX `name` (`name`),
INDEX `organisation_uuid` (`organisation_uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;

1
INSTALL/ansible/roles/misp/templates/misp/config/config.php
View File

@ -16,7 +16,6 @@ $config = array (
'org' => '',
'showorg' => true,
'background_jobs' => true,
'cached_attachments' => true,
'email' => '',
'contact' => '',
'cveurl' => 'http://cve.circl.lu/cve/',

2
PyMISP

@ -1 +1 @@
Subproject commit b212894152c9471e4da04eada9da887cad2f92a7
Subproject commit 8b66d5f75346271c78d4737f79fb302e17c961d2

16
README.md
View File

@ -108,16 +108,16 @@ License
This software is licensed under [GNU Affero General Public License version 3](http://www.gnu.org/licenses/agpl-3.0.html)
* Copyright (C) 2012-2020 Christophe Vandeplas
* Copyright (C) 2012-2022 Christophe Vandeplas
* Copyright (C) 2012 Belgian Defence
* Copyright (C) 2012 NATO / NCIRC
* Copyright (C) 2013-2020 Andras Iklody
* Copyright (C) 2015-2020 CIRCL - Computer Incident Response Center Luxembourg
* Copyright (C) 2013-2022 Andras Iklody
* Copyright (C) 2015-2022 CIRCL - Computer Incident Response Center Luxembourg
* Copyright (C) 2016 Andreas Ziegler
* Copyright (C) 2018-2020 Sami Mokaddem
* Copyright (C) 2018-2020 Christian Studer
* Copyright (C) 2015-2020 Alexandre Dulaunoy
* Copyright (C) 2018-2020 Steve Clement
* Copyright (C) 2020 Jakub Onderka
* Copyright (C) 2018-2022 Sami Mokaddem
* Copyright (C) 2018-2022 Christian Studer
* Copyright (C) 2015-2022 Alexandre Dulaunoy
* Copyright (C) 2018-2022 Steve Clement
* Copyright (C) 2020-2022 Jakub Onderka
For more information, [the list of authors and contributors](AUTHORS) is available.

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":150}
{"major":2, "minor":4, "hotfix":151}

15
app/Config/config.default.php
View File

@ -32,7 +32,6 @@ $config = array(
'email_subject_tag' => 'tlp',
'email_subject_include_tag_name' => true,
'background_jobs' => true,
'cached_attachments' => true,
'osuser' => 'www-data',
'email' => 'email@example.com',
'contact' => 'email@example.com',
@ -50,7 +49,6 @@ $config = array(
'unpublishedprivate' => false,
'disable_emailing' => false,
'manage_workers' => true,
'Attributes_Values_Filter_In_Event' => 'id, uuid, value, comment, type, category, Tag.name',
'python_bin' => null,
'external_baseurl' => '',
'forceHTTPSforPreLoginRequestedURL' => false,
@ -135,6 +133,19 @@ $config = array(
'amount' => 5,
'expire' => 300,
),
'SimpleBackgroundJobs' => array(
'enabled' => false,
'redis_host' => 'localhost',
'redis_port' => 6379,
'redis_password' => '',
'redis_database' => 1,
'redis_namespace' => 'background_jobs',
'max_job_history_ttl' => 86400,
'supervisor_host' => 'localhost',
'supervisor_port' => 9001,
'supervisor_user' => '',
'supervisor_password' => '',
),
// Uncomment the following to enable client SSL certificate authentication
/*
'CertAuth' => array(

5
app/Config/core.default.php
View File

@ -255,8 +255,9 @@ Configure::write('Acl.database', 'default');
* and their setttings.
*/
$engine = 'File';
if (extension_loaded('apc') && function_exists('apc_dec') && (php_sapi_name() !== 'cli' || ini_get('apc.enable_cli'))) {
$engine = 'Apc';
if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) {
require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin
$engine = 'Apcu'; // faster version of ApcEngine
}
// In development mode, caches should expire quickly.

34
app/Console/AppShell.php
View File

@ -1,34 +0,0 @@
<?php
/**
* AppShell file
*
* PHP 5
*
* CakePHP(tm) : Rapid Development Framework (http://cakephp.org)
* Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
*
* Licensed under The MIT License
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
* @link http://cakephp.org CakePHP(tm) Project
* @since CakePHP(tm) v 2.0
* @license MIT License (http://www.opensource.org/licenses/mit-license.php)
*/
App::uses('Shell', 'Console', 'AppModel', 'Model');
/**
* Application Shell
*
* Add your application-wide methods in the class below, your shells
* will inherit them.
*
* @package app.Console.Command
*/
class AppShell extends Shell {
public function perform() {
$this->initialize();
$this->{array_shift($this->args)}();
}
}

229
app/Console/Command/AdminShell.php
View File

@ -3,6 +3,7 @@ App::uses('AppShell', 'Console/Command');
/**
* @property Server $Server
* @property Feed $Feed
*/
class AdminShell extends AppShell
{
@ -22,12 +23,18 @@ class AdminShell extends AppShell
'value' => ['help' => __('Setting value'), 'required' => true],
],
'options' => [
'force' => array(
'force' => [
'short' => 'f',
'help' => 'Force the command.',
'default' => false,
'boolean' => true
)
],
'null' => [
'short' => 'n',
'help' => 'Set the value to null.',
'default' => false,
'boolean' => true
],
]
],
]);
@ -39,6 +46,27 @@ class AdminShell extends AppShell
],
],
]);
$parser->addSubcommand('reencrypt', [
'help' => __('Reencrypt encrypted values in database (authkeys and sensitive system settings).'),
'parser' => [
'options' => [
'old' => ['help' => __('Old key. If not provided, current key will be used.')],
'new' => ['help' => __('New key. If not provided, new key will be generated.')],
],
],
]);
$parser->addSubcommand('removeOrphanedCorrelations', [
'help' => __('Remove orphaned correlations.'),
]);
$parser->addSubcommand('optimiseTables', [
'help' => __('Optimise database tables.'),
]);
$parser->addSubcommand('redisMemoryUsage', [
'help' => __('Get detailed information about Redis memory usage.'),
]);
$parser->addSubcommand('redisReady', [
'help' => __('Check if it is possible connect to Redis.'),
]);
return $parser;
}
@ -369,14 +397,13 @@ class AdminShell extends AppShell
public function getSetting()
{
$this->ConfigLoad->execute();
$param = empty($this->args[0]) ? 'all' : $this->args[0];
$settings = $this->Server->serverSettingsRead();
$result = $settings;
if ($param != 'all') {
if ($param !== 'all') {
$result = 'No valid setting found for ' . $param;
foreach ($settings as $setting) {
if ($setting['setting'] == $param) {
if ($setting['setting'] === $param) {
$result = $setting;
break;
}
@ -387,15 +414,17 @@ class AdminShell extends AppShell
public function setSetting()
{
$setting_name = !isset($this->args[0]) ? null : $this->args[0];
$value = !isset($this->args[1]) ? null : $this->args[1];
list($setting_name, $value) = $this->args;
if ($value === 'false') {
$value = 0;
} elseif ($value === 'true') {
$value = 1;
}
if ($this->params['null']) {
$value = null;
}
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || $value === null) {
if (empty($setting_name) || ($value === null && !$this->params['null'])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
}
$setting = $this->Server->getSettingData($setting_name);
@ -405,7 +434,7 @@ class AdminShell extends AppShell
}
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
if ($result === true) {
echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL;
$this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
} else {
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
$this->error(__('Setting change rejected.'), $message);
@ -473,7 +502,7 @@ class AdminShell extends AppShell
{
try {
$redis = $this->Server->setupRedisWithException();
$redis->randomKey();
$redis->ping();
$this->out('Successfully connected to Redis.');
} catch (Exception $e) {
$this->error('Redis connection is not available', $e->getMessage());
@ -756,6 +785,33 @@ class AdminShell extends AppShell
$this->Job->saveField('status', 4);
}
public function removeOrphanedCorrelations()
{
$count = $this->Server->removeOrphanedCorrelations();
$this->out(__('%s orphaned correlation removed', $count));
}
public function optimiseTables()
{
$dataSource = $this->Server->getDataSource();
$tables = $dataSource->listSources();
/** @var ProgressShellHelper $progress */
$progress = $this->helper('progress');
$progress->init([
'total' => count($tables),
'width' => 50,
]);
foreach ($tables as $table) {
$dataSource->query('OPTIMISE TABLE ' . $dataSource->name($table));
$progress->increment();
$progress->draw();
}
$this->out('Optimised.');
}
public function updatesDone()
{
$blocking = !empty($this->args[0]);
@ -851,4 +907,157 @@ class AdminShell extends AppShell
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
}
}
public function reencrypt()
{
$old = $this->params['old'] ?? null;
$new = $this->params['new'] ?? null;
if ($new !== null && strlen($new) < 32) {
$this->error('New key must be at least 32 char long.');
}
if ($old === null) {
$old = Configure::read('Security.encryption_key');
}
if ($new === null) {
// Generate random new key
$randomTool = new RandomTool();
$new = $randomTool->random_str();
}
$this->Server->getDataSource()->begin();
try {
/** @var SystemSetting $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
$systemSetting->reencrypt($old, $new);
$this->Server->reencryptAuthKeys($old, $new);
/** @var Cerebrate $cerebrate */
$cerebrate = ClassRegistry::init('Cerebrate');
$cerebrate->reencryptAuthKeys($old, $new);
$result = $this->Server->serverSettingsSaveValue('Security.encryption_key', $new, true);
$this->Server->getDataSource()->commit();
if (!$result) {
$this->error('Encrypt key was changed, but it is not possible to save key to config file', __('Please insert new key "%s" to config file manually.', $new));
}
} catch (Exception $e) {
$this->Server->getDataSource()->rollback();
throw $e;
}
$this->out(__('New encryption key "%s" saved into config file.', $new));
}
/**
* @param Redis $redis
* @param string $prefix
* @return array[int, int]
*/
private function redisSize($redis, $prefix)
{
$keyCount = 0;
$size = 0;
$it = null;
while ($keys = $redis->scan($it, $prefix, 1000)) {
$redis->pipeline();
foreach ($keys as $key) {
$redis->rawCommand("memory", "usage", $key);
}
$result = $redis->exec();
$keyCount += count($keys);
$size += array_sum($result);
}
return [$keyCount, $size];
}
public function redisMemoryUsage()
{
$redis = $this->Server->setupRedisWithException();
$redis->setOption(Redis::OPT_SCAN, Redis::SCAN_RETRY);
$output = [];
list($count, $size) = $this->redisSize($redis, 'misp:feed_cache:*');
$output['feed_cache_count'] = $count;
$output['feed_cache_size'] = $size;
// Size of different feeds
$feedIds = $this->Feed->find('column', [
'fields' => ['id'],
]);
$redis->pipeline();
foreach ($feedIds as $feedId) {
$redis->rawCommand("memory", "usage", 'misp:feed_cache:' . $feedId);
}
$feedSizes = $redis->exec();
foreach ($feedIds as $k => $feedId) {
if ($feedSizes[$k]) {
$output['feed_cache_size_' . $feedId] = $feedSizes[$k];
}
}
list($count, $size) = $this->redisSize($redis, 'misp:server_cache:*');
$output['server_cache_count'] = $count;
$output['server_cache_size'] = $size;
// Size of different server
$serverIds = $this->Server->find('column', [
'fields' => ['id'],
]);
$redis->pipeline();
foreach ($serverIds as $serverId) {
$redis->rawCommand("memory", "usage", 'misp:server_cache:' . $serverId);
}
$serverSizes = $redis->exec();
foreach ($serverIds as $k => $serverId) {
if ($serverSizes[$k]) {
$output['server_cache_size_' . $serverId] = $serverSizes[$k];
}
}
list($count, $size) = $this->redisSize($redis, 'misp:wlc:*');
$output['warninglist_cache_count'] = $count;
$output['warninglist_cache_size'] = $size;
list($count, $size) = $this->redisSize($redis, 'misp:warninglist_entries_cache:*');
$output['warninglist_entries_count'] = $count;
$output['warninglist_entries_size'] = $size;
list($count, $size) = $this->redisSize($redis, 'misp:top_correlation');
$output['top_correlation_count'] = $count;
$output['top_correlation_size'] = $size;
list($count, $size) = $this->redisSize($redis, 'misp:correlation_exclusions');
$output['correlation_exclusions_count'] = $count;
$output['correlation_exclusions_size'] = $size;
list($count, $size) = $this->redisSize($redis, 'misp:event_lock:*');
$output['event_lock_count'] = $count;
$output['event_lock_size'] = $size;
list($count, $size) = $this->redisSize($redis, 'misp:user_ip:*');
$output['user_ip_count'] = $count;
$output['user_ip_size'] = $size;
list($count, $size) = $this->redisSize($redis, 'misp:ip_user:*');
$output['user_ip_count'] += $count;
$output['user_ip_size'] += $size;
list($count, $size) = $this->redisSize($redis, 'misp:authkey_usage:*');
$output['authkey_usage_count'] = $count;
$output['authkey_usage_size'] = $size;
$this->out($this->json($output));
}
}

9
app/Console/Command/AppShell.php
View File

@ -29,7 +29,14 @@ App::uses('AppModel', 'Model');
class AppShell extends Shell
{
public $tasks = array('ConfigLoad');
public function initialize()
{
parent::initialize();
$this->ConfigLoad = $this->Tasks->load('ConfigLoad');
$this->ConfigLoad->execute();
}
public function perform()
{
$this->initialize();

92
app/Console/Command/EventShell.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('FileAccessTool', 'Tools');
require_once 'AppShell.php';
/**
@ -141,28 +142,35 @@ class EventShell extends AppShell
$fieldList = array('published', 'id', 'info');
$this->Event->save($event, array('fieldList' => $fieldList));
// only allow form submit CSRF protection.
$this->Job->saveField('status', 1);
$this->Job->saveField('message', 'Job done.');
$this->Job->save([
'status' => Job::STATUS_COMPLETED,
'message' => 'Job done.'
]);
}
public function correlateValue()
{
$this->ConfigLoad->execute();
$value = $this->args[0];
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'correlateValue',
'job_input' => $value,
'status' => 0,
'retries' => 0,
'org' => 0,
'message' => 'Job created.',
);
$this->Job->save($data);
if (!empty($this->args[1])) {
$this->Job->id = intval($this->args[1]);
} else {
$this->Job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'correlateValue',
$value,
'Job created.'
);
}
$this->Correlation->correlateValue($value, $this->Job->id);
$this->Job->saveField('status', 1);
$this->Job->saveField('message', 'Job done.');
$this->Job->save([
'status' => Job::STATUS_COMPLETED,
'message' => 'Job done.',
'progress' => 100
]);
}
public function cache()
@ -299,23 +307,35 @@ class EventShell extends AppShell
public function postsemail()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) ||
empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5])) {
if (
empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) ||
empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5])
) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Posts email'] . PHP_EOL);
}
$userId = $this->args[0];
$postId = $this->args[1];
$eventId = $this->args[2];
$userId = intval($this->args[0]);
$postId = intval($this->args[1]);
$eventId = intval($this->args[2]);
$title = $this->args[3];
$message = $this->args[4];
$processId = $this->args[5];
$this->Job->id = $processId;
$this->Job->id = intval($this->args[5]);
$result = $this->Post->sendPostsEmail($userId, $postId, $eventId, $title, $message);
$job['Job']['progress'] = 100;
$job['Job']['message'] = 'Emails sent.';
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$this->Job->save($job);
if ($result) {
$this->Job->save([
'progress' => 100,
'message' => 'Emails sent.',
'date_modified' => date('Y-m-d H:i:s'),
'status' => Job::STATUS_COMPLETED
]);
} else {
$this->Job->save([
'date_modified' => date('Y-m-d H:i:s'),
'status' => Job::STATUS_FAILED
]);
}
}
public function enqueueCaching()
@ -391,6 +411,7 @@ class EventShell extends AppShell
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
$result = $this->Event->publish($id, $passAlong);
$job['Job']['progress'] = 100;
$job['Job']['status'] = Job::STATUS_COMPLETED;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
if ($result) {
$job['Job']['message'] = 'Event published.';
@ -404,7 +425,6 @@ class EventShell extends AppShell
public function publish_sightings()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[2]) || empty($this->args[3])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish sightings'] . PHP_EOL);
}
@ -414,14 +434,8 @@ class EventShell extends AppShell
$sightingsUuidsToPush = [];
if (isset($this->args[4])) { // push just specific sightings
$path = APP . 'tmp/cache/ingest' . DS . $this->args[4];
$tempFile = new File($path);
$inputData = $tempFile->read();
if ($inputData === false) {
$this->error("File `$path` not found.");
}
$sightingsUuidsToPush = $this->Event->jsonDecode($inputData);
$tempFile->delete();
$path = $this->args[4][0] === '/' ? $this->args[4] : (APP . 'tmp/cache/ingest' . DS . $this->args[4]);
$sightingsUuidsToPush = $this->Event->jsonDecode(FileAccessTool::readAndDelete($path));
}
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
@ -444,7 +458,7 @@ class EventShell extends AppShell
public function publish_galaxy_clusters()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL);
}
@ -528,9 +542,8 @@ class EventShell extends AppShell
$inputFile = $this->args[0];
$inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile;
$inputData = FileAccessTool::readFromFile($inputFile);
$inputData = FileAccessTool::readAndDelete($inputFile);
$inputData = $this->Event->jsonDecode($inputData);
FileAccessTool::deleteFile($inputFile);
Configure::write('CurrentUserId', $inputData['user']['id']);
$this->Event->processFreeTextData(
$inputData['user'],
@ -552,9 +565,8 @@ class EventShell extends AppShell
$inputFile = $this->args[0];
$inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile;
$inputData = FileAccessTool::readFromFile($inputFile);
$inputData = FileAccessTool::readAndDelete($inputFile);
$inputData = $this->Event->jsonDecode($inputData);
FileAccessTool::deleteFile($inputFile);
Configure::write('CurrentUserId', $inputData['user']['id']);
$this->Event->processModuleResultsData(
$inputData['user'],

108
app/Console/Command/ServerShell.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('BackgroundJobsTool', 'Tools');
require_once 'AppShell.php';
/**
@ -78,12 +79,19 @@ class ServerShell extends AppShell
));
foreach ($servers as $serverId => $serverName) {
$jobId = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $user['id'], $serverId, $technique)
$backgroundJobId = $this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'pull',
$user['id'],
$serverId,
$technique
]
);
$this->out("Enqueued pulling from $serverName server as job $jobId");
$this->out("Enqueued pulling from $serverName server as job $backgroundJobId");
}
}
@ -105,14 +113,14 @@ class ServerShell extends AppShell
if (!empty($this->args[3])) {
$jobId = $this->args[3];
} else {
$jobId = $this->Job->createJob($user,Job::WORKER_DEFAULT, 'pull', 'Server: ' . $serverId, 'Pulling.');
$jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'pull', 'Server: ' . $serverId, 'Pulling.');
}
$force = false;
if (!empty($this->args[4]) && $this->args[4] === 'force') {
$force = true;
}
try {
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
$result = $this->Server->pull($user, $technique, $server, $jobId, $force);
if (is_array($result)) {
$message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
$this->Job->saveStatus($jobId, true, $message);
@ -137,23 +145,12 @@ class ServerShell extends AppShell
$user = $this->getUser($userId);
$serverId = $this->args[1];
$server = $this->getServer($serverId);
if (!empty($this->args[2])) {
$jobId = $this->args[2];
$technique = empty($this->args[2]) ? 'full' : $this->args[2];
if (!empty($this->args[3])) {
$jobId = $this->args[3];
} else {
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'push',
'job_input' => 'Server: ' . $serverId,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
'message' => 'Pushing.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
$jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'push', 'Server: ' . $serverId, 'Pushing.');
}
$technique = empty($this->args[3]) ? 'full' : $this->args[3];
$this->Job->read(null, $jobId);
App::uses('SyncTool', 'Tools');
@ -192,11 +189,18 @@ class ServerShell extends AppShell
));
foreach ($servers as $serverId => $serverName) {
$jobId = CakeResque::enqueue(
'default',
'ServerShell',
array('push', $user['id'], $serverId, $technique)
$jobId = $this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'push',
$user['id'],
$serverId,
$technique
]
);
$this->out("Enqueued pushing from $serverName server as job $jobId");
}
}
@ -213,25 +217,13 @@ class ServerShell extends AppShell
if (!empty($this->args[2])) {
$jobId = $this->args[2];
} else {
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'fetch_feeds',
'job_input' => 'Feed: ' . $feedId,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
'message' => 'Starting fetch from Feed.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
$jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'fetch_feeds', 'Feed: ' . $feedId, 'Starting fetch from Feed.');
}
if ($feedId == 'all') {
$feedIds = $this->Feed->find('list', array(
'fields' => array('Feed.id', 'Feed.id'),
if ($feedId === 'all') {
$feedIds = $this->Feed->find('column', array(
'fields' => array('Feed.id'),
'conditions' => array('Feed.enabled' => 1)
));
$feedIds = array_values($feedIds);
$successes = 0;
$fails = 0;
foreach ($feedIds as $k => $feedId) {
@ -247,21 +239,21 @@ class ServerShell extends AppShell
$this->Job->saveStatus($jobId, true, $message);
echo $message . PHP_EOL;
} else {
$temp = $this->Feed->find('first', array(
'fields' => array('Feed.id', 'Feed.id'),
'conditions' => array('Feed.enabled' => 1, 'Feed.id' => $feedId)
));
if (!empty($temp)) {
$feedEnabled = $this->Feed->hasAny([
'Feed.enabled' => 1,
'Feed.id' => $feedId,
]);
if ($feedEnabled) {
$result = $this->Feed->downloadFromFeedInitiator($feedId, $user, $jobId);
if (!$result) {
$this->Job->saveStatus($jobId, false);
$this->Job->saveStatus($jobId, false, 'Job failed. See error log for more details.');
echo 'Job failed.' . PHP_EOL;
} else {
$this->Job->saveStatus($jobId, true);
echo 'Job done.' . PHP_EOL;
}
} else {
$message = "Feed with ID $feedId not found.";
$message = "Feed with ID $feedId not found or not enabled.";
$this->Job->saveStatus($jobId, false, $message);
echo $message . PHP_EOL;
}
@ -317,11 +309,17 @@ class ServerShell extends AppShell
));
foreach ($servers as $serverId => $serverName) {
$jobId = CakeResque::enqueue(
'default',
'ServerShell',
array('cacheServer', $user['id'], $serverId)
$jobId = $this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'cacheServer',
$user['id'],
$serverId
]
);
$this->out("Enqueued cacheServer from {$serverName} server as job $jobId");
}
}
@ -338,7 +336,7 @@ class ServerShell extends AppShell
if (!empty($this->args[2])) {
$jobId = $this->args[2];
} else {
$jobId = $this->Job->createJob($user,Job::WORKER_DEFAULT, 'cache_feeds', 'Feed: ' . $scope, 'Starting feed caching.');
$jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'cache_feeds', 'Feed: ' . $scope, 'Starting feed caching.');
}
try {
$result = $this->Feed->cacheFeedInitiator($user, $jobId, $scope);
@ -399,7 +397,7 @@ class ServerShell extends AppShell
);
$this->Job->save($data);
$jobId = $this->Job->id;
$result = $this->Server->pull($user, $server['Server']['id'], 'full', $server, $jobId);
$result = $this->Server->pull($user, 'full', $server, $jobId);
$this->Job->save(array(
'id' => $jobId,
'message' => 'Job done.',

104
app/Console/Command/StartWorkerShell.php Normal file
View File

@ -0,0 +1,104 @@
<?php
declare(strict_types=1);
App::uses('BackgroundJobsTool', 'Tools');
class StartWorkerShell extends AppShell
{
/** @var BackgroundJobsTool */
private $BackgroundJobsTool;
/** @var Worker */
private $worker;
/** @var int */
private $maxExecutionTime;
const DEFAULT_MAX_EXECUTION_TIME = 86400; // 1 day
public function initialize()
{
parent::initialize();
$this->BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs'));
}
public function getOptionParser(): ConsoleOptionParser
{
$parser = parent::getOptionParser();
$parser
->addArgument('queue', [
'help' => 'Name of the queue to process.',
'choices' => $this->BackgroundJobsTool->getQueues(),
'required' => true
])
->addOption(
'maxExecutionTime',
[
'help' => 'Worker maximum execution time (seconds) before it self-destruct. Zero means unlimited.',
'default' => self::DEFAULT_MAX_EXECUTION_TIME,
'required' => false
]
);
return $parser;
}
public function main()
{
$this->worker = new Worker(
[
'pid' => getmypid(),
'queue' => $this->args[0],
'user' => $this->whoami()
]
);
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
while (true) {
$this->checkMaxExecutionTime();
$job = $this->BackgroundJobsTool->dequeue($this->worker->queue());
if ($job) {
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
try {
$this->BackgroundJobsTool->run($job);
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");
$job->setStatus(BackgroundJob::STATUS_FAILED);
$this->BackgroundJobsTool->update($job);
}
}
}
}
/**
* Checks if worker maximum execution time is reached, and exits if so.
*
* @return void
*/
private function checkMaxExecutionTime()
{
if ($this->maxExecutionTime === 0) {
return;
}
if ((time() - $this->worker->createdAt()) > $this->maxExecutionTime) {
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - worker max execution time reached, exiting gracefully worker...");
exit;
}
}
private function whoami(): string
{
if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) {
return posix_getpwuid(posix_geteuid())['name'];
} else {
return trim(shell_exec('whoami'));
}
}
}

14
app/Console/Command/Task/ConfigLoadTask.php
View File

@ -1,7 +1,13 @@
<?php
class ConfigLoadTask extends Shell {
public function execute() {
Configure::load('config');
class ConfigLoadTask extends Shell
{
public function execute()
{
Configure::load('config');
if (Configure::read('MISP.system_setting_db')) {
App::uses('SystemSetting', 'Model');
SystemSetting::setGlobalSetting();
}
}
?>
}

39
app/Console/Command/UserShell.php
View File

@ -82,28 +82,31 @@ class UserShell extends AppShell
public function list()
{
// do not fetch sensitive or big values
$schema = $this->User->schema();
unset($schema['authkey']);
unset($schema['password']);
unset($schema['gpgkey']);
unset($schema['certif_public']);
$fields = array_keys($schema);
$fields[] = 'Role.*';
$fields[] = 'Organisation.*';
$users = $this->User->find('all', [
'recursive' => -1,
'fields' => $fields,
'contain' => ['Organisation', 'Role'],
]);
if ($this->params['json']) {
// do not fetch sensitive or big values
$schema = $this->User->schema();
unset($schema['authkey']);
unset($schema['password']);
unset($schema['gpgkey']);
unset($schema['certif_public']);
$fields = array_keys($schema);
$fields[] = 'Role.*';
$fields[] = 'Organisation.*';
$users = $this->User->find('all', [
'recursive' => -1,
'fields' => $fields,
'contain' => ['Organisation', 'Role', 'UserSetting'],
]);
$this->out($this->json($users));
} else {
$users = $this->User->find('column', [
'fields' => ['email'],
]);
foreach ($users as $user) {
$this->out($user['User']['email']);
$this->out($user);
}
}
}

109
app/Controller/AppController.php
View File

@ -31,10 +31,10 @@ class AppController extends Controller
*/
public $defaultModel = '';
public $helpers = array('OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '131';
public $pyMispVersion = '2.4.148';
public $pyMispVersion = '2.4.151';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
@ -69,34 +69,39 @@ class AppController extends Controller
}
public $components = array(
'Session',
'Auth' => array(
'authError' => 'Unauthorised access.',
'authenticate' => array(
'Form' => array(
'passwordHasher' => 'BlowfishConstant',
'fields' => array(
'username' => 'email'
)
'Session',
'Auth' => array(
'authError' => 'Unauthorised access.',
'authenticate' => array(
'Form' => array(
'passwordHasher' => 'BlowfishConstant',
'fields' => array(
'username' => 'email'
)
)
),
'Security',
'ACL',
'CompressedRequestHandler',
'RestResponse',
'Flash',
'Toolbox',
'RateLimit',
'IndexFilter',
'Deprecation',
'RestSearch',
'CRUD'
//,'DebugKit.Toolbar'
)
),
'Security',
'ACL',
'CompressedRequestHandler',
'RestResponse',
'Flash',
'Toolbox',
'RateLimit',
'IndexFilter',
'Deprecation',
'RestSearch',
'CRUD'
//,'DebugKit.Toolbar'
);
public function beforeFilter()
{
if (Configure::read('MISP.system_setting_db')) {
App::uses('SystemSetting', 'Model');
SystemSetting::setGlobalSetting();
}
$this->_setupBaseurl();
$this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
@ -132,7 +137,8 @@ class AppController extends Controller
$this->_setupDatabaseConnection();
$this->set('debugMode', Configure::read('debug') >= 1 ? 'debugOn' : 'debugOff');
$this->set('ajax', $this->request->is('ajax'));
$isAjax = $this->request->is('ajax');
$this->set('ajax', $isAjax);
$this->set('queryVersion', $this->__queryVersion);
$this->User = ClassRegistry::init('User');
@ -245,7 +251,7 @@ class AppController extends Controller
$this->__logAccess($user);
// Try to run updates
if ($user['Role']['perm_site_admin'] || (!$this->_isRest() && $this->_isLive())) {
if ($user['Role']['perm_site_admin'] || (!$this->_isRest() && !$isAjax && $this->_isLive())) {
$this->User->runUpdates();
}
@ -265,7 +271,7 @@ class AppController extends Controller
$user = $this->Auth->user(); // user info in session could change (see __verifyUser) method, so reload user variable
if (isset($user['logged_by_authkey']) && $user['logged_by_authkey'] && !($this->_isRest() || $this->_isAutomation())) {
throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed");
throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed.");
}
// Put token expiration time to response header that can be processed by automation tool
@ -321,12 +327,15 @@ class AppController extends Controller
$preAuthActions[] = 'email_otp';
}
if (!$this->_isControllerAction(['users' => $preAuthActions, 'servers' => ['cspReport']])) {
if (!$this->request->is('ajax')) {
if ($isAjax) {
$response = $this->RestResponse->throwException(401, "Unauthorized");
$response->send();
$this->_stop();
} else {
$this->Session->write('pre_login_requested_url', $this->request->here);
$this->_redirectToLogin();
}
$this->_redirectToLogin();
}
$this->set('me', false);
}
@ -338,7 +347,7 @@ class AppController extends Controller
$this->User->Server->updateDatabase('cleanSessionTable');
}
}
if (Configure::read('site_admin_debug') && (Configure::read('debug') < 2)) {
if (Configure::read('site_admin_debug') && Configure::read('debug') < 2) {
Configure::write('debug', 1);
}
}
@ -361,19 +370,15 @@ class AppController extends Controller
}
// Notifications and homepage is not necessary for AJAX or REST requests
if ($user && !$this->_isRest() && !$this->request->is('ajax')) {
if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') {
$notifications = $this->User->populateNotifications($user);
} else {
$notifications = $this->User->populateNotifications($user, 'fast');
}
$this->set('notifications', $notifications);
if ($user && !$this->_isRest() && !$isAjax) {
$hasNotifications = $this->User->hasNotifications($user);
$this->set('hasNotifications', $hasNotifications);
$homepage = $this->User->UserSetting->getValueForUser($user['id'], 'homepage');
if (!empty($homepage)) {
$this->set('homepage', $homepage);
}
if (version_compare(phpversion(), '8.0') >= 0) {
if (PHP_MAJOR_VERSION >= 8) {
$this->Flash->error(__('WARNING: MISP is currently running under PHP 8.0, which is unsupported. Background jobs will fail, so please contact your administrator to run a supported PHP version (such as 7.4)'));
}
}
@ -906,22 +911,29 @@ class AppController extends Controller
return $user;
}
// generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
protected function _harvestParameters($options, &$exception, $data = array())
/**
* generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
* @param array $options
* @param $exception
* @param array $data
* @return array|false|mixed
*/
protected function _harvestParameters($options, &$exception = null, $data = array())
{
if (!empty($options['request']->is('post'))) {
if (empty($options['request']->data)) {
$request = $options['request'] ?? $this->request;
if ($request->is('post')) {
if (empty($request->data)) {
$exception = $this->RestResponse->throwException(
400,
__('Either specify the search terms in the url, or POST a json with the filter parameters.'),
'/' . $this->request->params['controller'] . '/' . $this->request->action
'/' . $request->params['controller'] . '/' . $request->action
);
return false;
} else {
if (isset($options['request']->data['request'])) {
$data = array_merge($data, $options['request']->data['request']);
if (isset($request->data['request'])) {
$data = array_merge($data, $request->data['request']);
} else {
$data = array_merge($data, $options['request']->data);
$data = array_merge($data, $request->data);
}
}
}
@ -1224,6 +1236,9 @@ class AppController extends Controller
if ($user === false) {
return $exception;
}
session_write_close(); // Rest search can be longer, so close session to allow concurrent requests
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
if ($returnFormat === 'download') {

254
app/Controller/AttributesController.php
View File

@ -11,12 +11,23 @@ class AttributesController extends AppController
{
public $components = array('Security', 'RequestHandler');
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999,
'conditions' => array('AND' => array('Attribute.deleted' => 0)),
'order' => 'Attribute.event_id DESC'
);
public $paginate = [
'limit' => 60,
'maxLimit' => 9999,
'conditions' => array('AND' => array('Attribute.deleted' => 0)),
'order' => 'Attribute.event_id DESC',
'recursive' => -1,
'contain' => array(
'Event' => array(
'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
),
'AttributeTag',
'Object' => array(
'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
),
'SharingGroup' => ['fields' => ['SharingGroup.name']],
),
];
public function beforeFilter()
{
@ -53,36 +64,19 @@ class AttributesController extends AppController
$this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
}
}
// do not show private to other orgs
if (!$this->_isSiteAdmin()) {
$this->paginate = Set::merge($this->paginate, array('conditions' => $this->Attribute->buildConditions($this->Auth->user())));
}
}
public function index()
{
$this->Attribute->recursive = -1;
$this->paginate['recursive'] = -1;
$this->paginate['contain'] = array(
'Event' => array(
'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
),
'AttributeTag' => array('Tag'),
'Object' => array(
'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
),
'SharingGroup' => ['fields' => ['SharingGroup.name']],
);
$this->Attribute->contain(array('AttributeTag' => array('Tag')));
$this->set('isSearch', 0);
$this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($this->Auth->user());
$attributes = $this->paginate();
if ($this->_isRest()) {
foreach ($attributes as $k => $attribute) {
$attributes[$k] = $attribute['Attribute'];
}
$attributes = array_column($attributes, 'Attribute');
return $this->RestResponse->viewData($attributes, $this->response->type());
}
$this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
$orgTable = $this->Attribute->Event->Orgc->find('all', [
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
]);
@ -94,6 +88,7 @@ class AttributesController extends AppController
}
list($attributes, $sightingsData) = $this->__searchUI($attributes);
$this->set('isSearch', 0);
$this->set('sightingsData', $sightingsData);
$this->set('orgTable', array_column($orgTable, 'name', 'id'));
$this->set('shortDist', $this->Attribute->shortDist);
@ -920,28 +915,25 @@ class AttributesController extends AppController
if (empty($attribute)) {
return new CakeResponse(array('body'=> json_encode(array('fail' => false, 'errors' => 'Invalid attribute')), 'status' => 200, 'type' => 'json'));
}
$this->Attribute->data = $attribute;
$this->Attribute->id = $attribute['Attribute']['id'];
if (!$this->__canModifyEvent($attribute)) {
return new CakeResponse(array('body' => json_encode(array('fail' => false, 'errors' => 'You do not have permission to do that')), 'status' => 200, 'type' => 'json'));
}
if (!$this->_isRest()) {
$this->Attribute->Event->insertLock($this->Auth->user(), $attribute['Attribute']['event_id']);
}
$validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen');
$changed = false;
if (empty($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
if (empty($this->request->data['Attribute'])) {
throw new MethodNotAllowedException(__('Invalid input.'));
}
}
$validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen');
$changed = false;
foreach ($this->request->data['Attribute'] as $changedKey => $changedField) {
if (!in_array($changedKey, $validFields)) {
if (!in_array($changedKey, $validFields, true)) {
throw new MethodNotAllowedException(__('Invalid field.'));
}
if ($attribute['Attribute'][$changedKey] == $changedField) {
$this->autoRender = false;
return new CakeResponse(array('body'=> json_encode(array('errors'=> array('value' => 'nochange'))), 'status'=>200, 'type' => 'json'));
}
$attribute['Attribute'][$changedKey] = $changedField;
@ -952,16 +944,23 @@ class AttributesController extends AppController
}
$date = new DateTime();
$attribute['Attribute']['timestamp'] = $date->getTimestamp();
if ($this->Attribute->save($attribute)) {
$this->Attribute->Event->unpublishEvent($attribute['Attribute']['event_id']);
$fieldsToSave = ['timestamp'];
if ($changedKey === 'value') {
$fieldsToSave[] = 'value1';
$fieldsToSave[] = 'value2';
} else {
$fieldsToSave[] = $changedKey;
}
if ($this->Attribute->save($attribute, true, $fieldsToSave)) {
$this->Attribute->Event->unpublishEvent($attribute['Attribute']['event_id'], false, $date->getTimestamp());
if ($attribute['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
}
$this->autoRender = false;
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.', 'check_publish' => true)), 'status'=>200, 'type' => 'json'));
} else {
$this->autoRender = false;
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $this->Attribute->validationErrors)), 'status'=>200, 'type' => 'json'));
}
}
@ -1577,25 +1576,9 @@ class AttributesController extends AppController
if (!isset($params['conditions']['Attribute.deleted'])) {
$params['conditions']['Attribute.deleted'] = 0;
}
$this->paginate = $params;
if (empty($this->paginate['limit'])) {
$this->paginate['limit'] = 60;
}
if (empty($this->paginate['page'])) {
$this->paginate['page'] = 1;
}
$this->paginate['recursive'] = -1;
$this->paginate['contain'] = array(
'Event' => array(
'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
),
'AttributeTag' => array('Tag'),
'Object' => array(
'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
),
'SharingGroup' => ['fields' => ['SharingGroup.name']],
);
$this->paginate['conditions'] = $params['conditions'];
$attributes = $this->paginate();
$this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
$orgTable = $this->Attribute->Event->Orgc->find('all', [
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
@ -1649,7 +1632,7 @@ class AttributesController extends AppController
}
}
private function __searchUI($attributes)
private function __searchUI(array $attributes)
{
if (empty($attributes)) {
return [[], []];
@ -1661,9 +1644,9 @@ class AttributesController extends AppController
$this->loadModel('AttachmentScan');
$user = $this->Auth->user();
$attributeIds = [];
foreach ($attributes as $k => $attribute) {
$attributeId = $attribute['Attribute']['id'];
$attributeIds[] = $attributeId;
$galaxyTags = [];
foreach ($attributes as &$attribute) {
$attributeIds[] = $attribute['Attribute']['id'];
if ($this->Attribute->isImage($attribute['Attribute'])) {
if (extension_loaded('gd')) {
// if extension is loaded, the data is not passed to the view because it is asynchronously fetched
@ -1671,20 +1654,33 @@ class AttributesController extends AppController
} else {
$attribute['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']);
}
$attributes[$k] = $attribute;
}
if ($attribute['Attribute']['type'] === 'attachment' && $this->AttachmentScan->isEnabled()) {
$infected = $this->AttachmentScan->isInfected(AttachmentScan::TYPE_ATTRIBUTE, $attribute['Attribute']['id']);
$attributes[$k]['Attribute']['infected'] = $infected;
$attribute['Attribute']['infected'] = $infected;
}
if ($attribute['Attribute']['distribution'] == 4) {
$attributes[$k]['Attribute']['SharingGroup'] = $attribute['SharingGroup'];
$attribute['Attribute']['SharingGroup'] = $attribute['SharingGroup'];
}
$attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($this->Auth->user(), $attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true);
unset($attributes[$k]['AttributeTag']);
$attribute['Attribute']['AttributeTag'] = $attribute['AttributeTag'];
foreach ($attribute['Attribute']['AttributeTag'] as $at) {
if (substr($at['Tag']['name'], 0, 12) === 'misp-galaxy:') {
$galaxyTags[] = $at['Tag']['name'];
}
}
unset($attribute['AttributeTag']);
}
unset($attribute);
// Fetch galaxy clusters in one query
if (!empty($galaxyTags)) {
$this->loadModel('GalaxyCluster');
$clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false);
$clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
} else {
$clusters = [];
}
// Fetch correlations in one query
@ -1696,6 +1692,27 @@ class AttributesController extends AppController
$attributesWithFeedCorrelations = $this->Feed->attachFeedCorrelations(array_column($attributes, 'Attribute'), $user, $fakeEventArray);
foreach ($attributes as $k => $attribute) {
// Assign galaxies
$galaxies = [];
foreach ($attribute['Attribute']['AttributeTag'] as $k2 => $attributeTag) {
if (!isset($clusters[$attributeTag['Tag']['id']])) {
continue;
}
$cluster = $clusters[$attributeTag['Tag']['id']];
$galaxyId = $cluster['Galaxy']['id'];
$cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
if (isset($attribute['Attribute']['Galaxy'][$galaxyId])) {
unset($cluster['Galaxy']);
$galaxies[$galaxyId]['GalaxyCluster'][] = $cluster;
} else {
$galaxies[$galaxyId] = $cluster['Galaxy'];
unset($cluster['Galaxy']);
$galaxies[$galaxyId]['GalaxyCluster'] = [$cluster];
}
unset($attributes[$k]['Attribute']['AttributeTag'][$k2]); // remove galaxy tag
}
$attributes[$k]['Attribute']['Galaxy'] = array_values($galaxies);
if (isset($attributesWithFeedCorrelations[$k]['Feed'])) {
$attributes[$k]['Attribute']['Feed'] = $attributesWithFeedCorrelations[$k]['Feed'];
}
@ -1707,65 +1724,6 @@ class AttributesController extends AppController
return array($attributes, $sightingsData);
}
// If the checkbox for the alternate search is ticked, then this method is called to return the data to be represented
// This alternate view will show a list of events with matching search results and the percentage of those matched attributes being marked as to_ids
// events are sorted based on relevance (as in the percentage of matches being flagged as indicators for IDS)
public function searchAlternate($data)
{
$attributes = $this->Attribute->fetchAttributes(
$this->Auth->user(),
array(
'conditions' => array(
'AND' => $data
),
'contain' => array('Event' => array('Orgc' => array('fields' => array('Orgc.name')))),
'fields' => array(
'Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids', 'Attribute.value', 'Attribute.distribution',
'Event.id', 'Event.org_id', 'Event.orgc_id', 'Event.info', 'Event.distribution', 'Event.attribute_count', 'Event.date',
)
)
);
$events = array();
foreach ($attributes as $attribute) {
if (isset($events[$attribute['Event']['id']])) {
if ($attribute['Attribute']['to_ids']) {
$events[$attribute['Event']['id']]['to_ids']++;
} else {
$events[$attribute['Event']['id']]['no_ids']++;
}
} else {
$events[$attribute['Event']['id']]['Event'] = $attribute['Event'];
$events[$attribute['Event']['id']]['to_ids'] = 0;
$events[$attribute['Event']['id']]['no_ids'] = 0;
if ($attribute['Attribute']['to_ids']) {
$events[$attribute['Event']['id']]['to_ids']++;
} else {
$events[$attribute['Event']['id']]['no_ids']++;
}
}
}
foreach ($events as $key => $event) {
$events[$key]['relevance'] = 100 * $event['to_ids'] / ($event['no_ids'] + $event['to_ids']);
}
if (!empty($events)) {
$events = $this->__subval_sort($events, 'relevance');
}
return $events;
}
// Sort the array of arrays based on a value of a sub-array
private function __subval_sort($a, $subkey)
{
foreach ($a as $k=>$v) {
$b[$k] = strtolower($v[$subkey]);
}
arsort($b);
foreach ($b as $key=>$val) {
$c[] = $a[$key];
}
return $c;
}
public function checkComposites()
{
if (!self::_isAdmin()) {
@ -1782,7 +1740,7 @@ class AttributesController extends AppController
if (!$this->Auth->user()) {
throw new UnauthorizedException(__('You are not authorized. Please send the Authorization header with your auth key along with an Accept header for application/xml.'));
}
$user = $this->_checkAuthUser($this->Auth->user('authkey'));
$user = $this->Auth->user();
}
// if the user is authorised to use the api key then user will be populated with the user's account
// in addition we also set a flag indicating whether the user is a site admin or not.
@ -1960,26 +1918,27 @@ class AttributesController extends AppController
$this->Flash->success(__('All done. ' . $k . ' attributes processed.'));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
} else {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'default',
'job_type' => 'generate correlation',
'job_input' => 'All attributes',
'status' => 0,
'retries' => 0,
'org' => 'ADMIN',
'message' => 'Job created.',
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate correlation',
'All attributes',
'Job created.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('jobGenerateCorrelation', $jobId),
true
$this->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateCorrelation',
$jobId
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).'));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
}
@ -2673,16 +2632,11 @@ class AttributesController extends AppController
}
}
} else {
$conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id)));
if (!$this->_isSiteAdmin()) {
$conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id'));
$conditions['Tag.user_id'] = array('0', $this->Auth->user('id'));
}
$tag = $this->Attribute->AttributeTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions));
if (empty($tag)) {
$tagId = $this->Attribute->AttributeTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id));
if (empty($tagId)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json'));
}
$tag_id = $tag['Tag']['id'];
$tag_id = $tagId;
}
}
}

19
app/Controller/AuditLogsController.php
View File

@ -37,6 +37,7 @@ class AuditLogsController extends AppController
'Server',
'ShadowAttribute',
'SharingGroup',
'SystemSetting',
'Tag',
'TagCollection',
'TagCollectionTag',
@ -484,10 +485,12 @@ class AuditLogsController extends AppController
if (!empty($eventIds)) {
$this->loadModel('Event');
$events = $this->Event->fetchSimpleEvents($this->Auth->user(), [
'conditions' => ['Event.id' => array_unique($eventIds)],
$conditions = $this->Event->createEventConditions($this->Auth->user());
$conditions['Event.id'] = array_unique($eventIds);
$events = $this->Event->find('list', [
'conditions' => $conditions,
'fields' => ['Event.id', 'Event.info'],
]);
$events = array_column(array_column($events, 'Event'), null, 'id');
}
$links = [
@ -525,7 +528,7 @@ class AuditLogsController extends AppController
case 'Event':
if (isset($events[$modelId])) {
$url = '/events/view/' . $modelId;
$eventInfo = $events[$modelId]['info'];
$eventInfo = $events[$modelId];
}
break;
case 'ObjectReference':
@ -535,7 +538,7 @@ class AuditLogsController extends AppController
$url .= '/deleted:2';
}
if (isset($events[$objects[$objectReferences[$modelId]]['event_id']])) {
$eventInfo = $events[$objects[$objectReferences[$modelId]]['event_id']]['info'];
$eventInfo = $events[$objects[$objectReferences[$modelId]]['event_id']];
}
}
break;
@ -546,7 +549,7 @@ class AuditLogsController extends AppController
$url .= '/deleted:2';
}
if (isset($events[$objects[$modelId]['event_id']])) {
$eventInfo = $events[$objects[$modelId]['event_id']]['info'];
$eventInfo = $events[$objects[$modelId]['event_id']];
}
}
break;
@ -557,7 +560,7 @@ class AuditLogsController extends AppController
$url .= '/deleted:2';
}
if (isset($events[$attributes[$modelId]['event_id']])) {
$eventInfo = $events[$attributes[$modelId]['event_id']]['info'];
$eventInfo = $events[$attributes[$modelId]['event_id']];
}
}
break;
@ -565,7 +568,7 @@ class AuditLogsController extends AppController
if (isset($shadowAttributes[$modelId])) {
$url = '/events/view/' . $shadowAttributes[$modelId]['event_id'] . '/focus:' . $shadowAttributes[$modelId]['uuid'];
if (isset($events[$shadowAttributes[$modelId]['event_id']])) {
$eventInfo = $events[$shadowAttributes[$modelId]['event_id']]['info'];
$eventInfo = $events[$shadowAttributes[$modelId]['event_id']];
}
}
break;

1
app/Controller/Component/ACLComponent.php
View File

@ -54,7 +54,6 @@ class ACLComponent extends Component
'returnAttributes' => array('*'),
'rpz' => array('*'),
'search' => array('*'),
'searchAlternate' => array('*'),
'toggleCorrelation' => array('perm_add'),
'text' => array('*'),
'toggleToIDS' => array('perm_add'),

9
app/Controller/Component/RestResponseComponent.php
View File

@ -512,6 +512,7 @@ class RestResponseComponent extends Component
* @param bool $download
* @param array $headers
* @return CakeResponse
* @throws Exception
*/
private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
{
@ -535,7 +536,7 @@ class RestResponseComponent extends Component
$type = 'xml';
} elseif ($format === 'openioc') {
$type = 'xml';
} elseif ($format === 'csv') {
} elseif ($format === 'csv' || $format === 'text/csv') {
$type = 'csv';
} else {
if (empty($format)) {
@ -582,6 +583,12 @@ class RestResponseComponent extends Component
}
App::uses('TmpFileTool', 'Tools');
if ($response instanceof Generator) {
$tmpFile = new TmpFileTool();
$tmpFile->writeWithSeparator($response, null);
$response = $tmpFile;
}
if ($response instanceof TmpFileTool) {
App::uses('CakeResponseTmp', 'Tools');
$cakeResponse = new CakeResponseTmp(['status' => $code, 'type' => $type]);

752
app/Controller/EventsController.php
View File

File diff suppressed because it is too large Load Diff

142
app/Controller/FeedsController.php
View File

@ -488,26 +488,30 @@ class FeedsController extends AppController
}
}
if (Configure::read('MISP.background_jobs')) {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'fetch_feeds',
'job_input' => 'Feed: ' . $feedId,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Starting fetch from Feed.'),
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'fetch_feeds',
'Feed: ' . $feedId,
__('Starting fetch from Feed.')
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('fetchFeed', $this->Auth->user('id'), $feedId, $jobId),
true
$this->Feed->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'fetchFeed',
$this->Auth->user('id'),
$feedId,
$jobId
],
true,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$message = __('Pull queued for background execution.');
} else {
$result = $this->Feed->downloadFromFeedInitiator($feedId, $this->Auth->user());
@ -554,26 +558,30 @@ class FeedsController extends AppController
continue;
}
if (Configure::read('MISP.background_jobs')) {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'fetch_feed',
'job_input' => 'Feed: ' . $feedId,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Starting fetch from Feed.'),
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'fetch_feed',
'Feed: ' . $feedId,
__('Starting fetch from Feed.')
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('fetchFeed', $this->Auth->user('id'), $feedId, $jobId),
true
$this->Feed->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'fetchFeed',
$this->Auth->user('id'),
$feedId,
$jobId
],
true,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$message = 'Pull queued for background execution.';
} else {
$result = $this->Feed->downloadFromFeedInitiator($feedId, $this->Auth->user());
@ -747,21 +755,31 @@ class FeedsController extends AppController
} else {
$currentPage = 1;
}
$urlparams = '';
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
if (!in_array($feed['Feed']['source_format'], array('freetext', 'csv'))) {
throw new MethodNotAllowedException(__('Invalid feed type.'));
}
$HttpSocket = $syncTool->setupHttpSocketFeed($feed);
$params = array();
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocketFeed();
// params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it
try {
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage, 60, $params);
$resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']);
} catch (Exception $e) {
$this->Flash->error("Could not fetch feed: {$e->getMessage()}");
$this->redirect(array('controller' => 'feeds', 'action' => 'index'));
}
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$params = $customPagination->createPaginationRules($resultArray, array('page' => $currentPage, 'limit' => 60), 'Feed', $sort = false);
if (!empty($currentPage) && $currentPage !== 'all') {
$start = ($currentPage - 1) * 60;
if ($start > count($resultArray)) {
return false;
}
$resultArray = array_slice($resultArray, $start, 60);
}
$this->params->params['paging'] = array($this->modelClass => $params);
$resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray, $feed['Feed']['id']);
// remove all duplicates
@ -923,26 +941,30 @@ class FeedsController extends AppController
public function cacheFeeds($scope = 'freetext')
{
if (Configure::read('MISP.background_jobs')) {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'cache_feeds',
'job_input' => $scope,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Starting feed caching.'),
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'cache_feeds',
$scope,
__('Starting feed caching.')
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('cacheFeed', $this->Auth->user('id'), $scope, $jobId),
true
$this->Feed->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'cacheFeed',
$this->Auth->user('id'),
$scope,
$jobId
],
true,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$message = 'Feed caching job initiated.';
} else {
$result = $this->Feed->cacheFeedInitiator($this->Auth->user(), false, $scope);

8
app/Controller/GalaxiesController.php
View File

@ -339,6 +339,11 @@ class GalaxiesController extends AppController
$conditions[] = [
'enabled' => true
];
if(!$local) {
$conditions[] = [
'local_only' => 0
];
}
$galaxies = $this->Galaxy->find('all', array(
'recursive' => -1,
'fields' => array('MAX(Galaxy.version) as latest_version', 'id', 'kill_chain_order', 'name', 'icon', 'description'),
@ -528,8 +533,9 @@ class GalaxiesController extends AppController
public function attachCluster($target_id, $target_type = 'event')
{
$local = !empty($this->params['named']['local']);
$cluster_id = $this->request->data['Galaxy']['target_id'];
$result = $this->Galaxy->attachCluster($this->Auth->user(), $target_type, $target_id, $cluster_id);
$result = $this->Galaxy->attachCluster($this->Auth->user(), $target_type, $target_id, $cluster_id, $local);
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $result, 'check_publish' => true)), 'status'=>200, 'type' => 'json'));
}

50
app/Controller/JobsController.php
View File

@ -6,7 +6,7 @@ App::uses('AppController', 'Controller');
*/
class JobsController extends AppController
{
public $components = array('Security' ,'RequestHandler', 'Session');
public $components = array('Security', 'RequestHandler', 'Session');
public $paginate = array(
'limit' => 20,
@ -30,13 +30,18 @@ class JobsController extends AppController
}
$jobs = $this->paginate();
foreach ($jobs as &$job) {
if ($job['Job']['process_id'] !== false) {
$job['Job']['job_status'] = $this->__jobStatusConverter(CakeResque::getJobStatus($job['Job']['process_id']));
if (!empty($job['Job']['process_id'])) {
$job['Job']['job_status'] = $this->__getJobStatus($job['Job']['process_id']);
$job['Job']['failed'] = $job['Job']['job_status'] === 'Failed';
} else {
$job['Job']['job_status'] = 'Unknown';
$job['Job']['failed'] = null;
}
if(Configure::read('SimpleBackgroundJobs.enabled')){
$job['Job']['worker_status'] = true;
}else{
$job['Job']['worker_status'] = isset($workers[$job['Job']['worker']]) && $workers[$job['Job']['worker']]['ok'];
}
$job['Job']['worker_status'] = isset($workers[$job['Job']['worker']]) && $workers[$job['Job']['worker']]['ok'];
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($jobs, $this->response->type());
@ -53,7 +58,7 @@ class JobsController extends AppController
'Error' => 'error'
);
$this->set('fields', $fields);
$this->set('response', CakeResque::getFailedJobLog($id));
$this->set('response', $this->__getFailedJobLog($id));
$this->render('/Jobs/ajax/error');
}
@ -84,7 +89,7 @@ class JobsController extends AppController
throw new NotFoundException("Job with ID `$id` not found");
}
$output = [
'job_status' => $this->__jobStatusConverter(CakeResque::getJobStatus($job['Job']['process_id'])),
'job_status' => $this->__getJobStatus($job['Job']['process_id']),
'progress' => (int)$job['Job']['progress'],
];
return $this->RestResponse->viewData($output, 'json');
@ -136,7 +141,7 @@ class JobsController extends AppController
if ($this->_isSiteAdmin()) {
$target = 'All events.';
} else {
$target = 'Events visible to: '.$this->Auth->user('Organisation')['name'];
$target = 'Events visible to: ' . $this->Auth->user('Organisation')['name'];
}
$id = $this->Job->cache($type, $this->Auth->user());
if ($this->_isRest()) {
@ -161,4 +166,35 @@ class JobsController extends AppController
$this->redirect(array('action' => 'index'));
}
}
private function __getJobStatus(?string $id): string
{
if (!Configure::read('SimpleBackgroundJobs.enabled')) {
return $this->__jobStatusConverter(CakeResque::getJobStatus($id));
}
$status = null;
if (!empty($id)) {
$job = $this->Job->getBackgroundJobsTool()->getJob($id);
$status = $job ? $job->status() : $status;
}
return $this->__jobStatusConverter($status);
}
private function __getFailedJobLog(string $id): array
{
if (!Configure::read('SimpleBackgroundJobs.enabled')) {
return CakeResque::getFailedJobLog($id);
}
$job = $this->Job->getBackgroundJobsTool()->getJob($id);
$output = $job ? $job->output() : __('Job status not found.');
$backtrace = $job ? explode("\n", $job->error()) : [];
return [
'error' => $output ?? $backtrace[0] ?? '',
'backtrace' => $backtrace
];
}
}

8
app/Controller/ObjectReferencesController.php
View File

@ -70,7 +70,7 @@ class ObjectReferencesController extends AppController
$this->ObjectReference->create();
$result = $this->ObjectReference->save(array('ObjectReference' => $data));
if ($result) {
$this->ObjectReference->updateTimestamps($this->id, $data);
$this->ObjectReference->updateTimestamps($data);
if ($this->_isRest()) {
$object = $this->ObjectReference->find("first", array(
'recursive' => -1,
@ -203,10 +203,10 @@ class ObjectReferencesController extends AppController
throw new NotFoundException(__('Invalid object reference.'));
}
// Check if user can view object that contains this reference
$object = $this->ObjectReference->Object->find($this->Auth->user(), [
'conditions' => $objectReference['ObjectReference']['object_id'],
$object = $this->ObjectReference->Object->fetchObjectSimple($this->Auth->user(), [
'conditions' => ['Object.id' => $objectReference['ObjectReference']['object_id']],
]);
if (!$object) {
if (empty($object)) {
throw new NotFoundException(__('Invalid object reference.'));
}
return $this->RestResponse->viewData($objectReference, 'json');

6
app/Controller/ObjectTemplatesController.php
View File

@ -57,8 +57,7 @@ class ObjectTemplatesController extends AppController
$templates_raw = $this->ObjectTemplate->find('all', array(
'recursive' => -1,
'conditions' => $conditions,
'fields' => array('id', 'meta-category', 'name', 'description', 'org_id'),
'contain' => array('Organisation.name'),
'fields' => array('id', 'meta-category', 'name', 'description'),
'order' => array('ObjectTemplate.name asc')
));
@ -76,10 +75,9 @@ class ObjectTemplatesController extends AppController
);
}
$fun = 'redirectAddObject';
$this->set('items', $items);
$this->set('options', array(
'functionName' => $fun,
'functionName' => 'redirectAddObject',
'multiple' => 0,
'select_options' => array(
'additionalData' => array('event_id' => $event_id),

27
app/Controller/OrganisationsController.php
View File

@ -372,19 +372,15 @@ class OrganisationsController extends AppController
}
$idList = json_decode($idList, true);
$id_exclusion_list = array_merge($idList, array($this->Auth->user('Organisation')['id']));
$temp = $this->Organisation->find('all', array(
'conditions' => array(
'local' => $local,
'id !=' => $id_exclusion_list,
),
'recursive' => -1,
'fields' => array('id', 'name'),
'order' => array('lower(name) ASC')
$orgs = $this->Organisation->find('list', array(
'conditions' => array(
'local' => $local,
'id !=' => $id_exclusion_list,
),
'recursive' => -1,
'fields' => array('id', 'name'),
'order' => array('lower(name) ASC')
));
$orgs = array();
foreach ($temp as $org) {
$orgs[] = array('id' => $org['Organisation']['id'], 'name' => $org['Organisation']['name']);
}
$this->set('local', $local);
$this->layout = false;
$this->autoRender = false;
@ -402,10 +398,13 @@ class OrganisationsController extends AppController
$this->render('ajax/sg_org_row_empty');
}
/**
* @deprecated Probably not used anywhere.
*/
public function getUUIDs()
{
if (!$this->Auth->user('Role')['perm_sync']) {
throw new MethodNotAllowedException(__('This action is restricted to sync users'));
if (Configure::read('Security.hide_organisation_index_from_users')) {
throw new MethodNotAllowedException(__('This action is not enabled on this instance.'));
}
$temp = $this->Organisation->find('all', array(
'recursive' => -1,

347
app/Controller/ServersController.php
View File

@ -126,8 +126,16 @@ class ServersController extends AppController
try {
list($events, $total_count) = $this->Server->previewIndex($server, $this->Auth->user(), $combinedArgs);
} catch (Exception $e) {
$this->Flash->error(__('Download failed.') . ' ' . $e->getMessage());
$this->redirect(array('action' => 'index'));
if ($this->_isRest()) {
return $this->RestResponse->throwException(500, $e->getMessage());
} else {
$this->Flash->error(__('Download failed.') . ' ' . $e->getMessage());
$this->redirect(array('action' => 'index'));
}
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($events, $this->response->type());
}
$this->loadModel('Event');
@ -704,32 +712,36 @@ class ServersController extends AppController
* incremental - only new events
* <int> - specific id of the event to pull
*/
public function pull($id = null, $technique='full')
public function pull($id = null, $technique = 'full')
{
if (!empty($id)) {
$this->Server->id = $id;
} else if (!empty($this->request->data['id'])) {
$this->Server->id = $this->request->data['id'];
} else {
if (empty($id)) {
if (!empty($this->request->data['id'])) {
$id = $this->request->data['id'];
} else {
throw new NotFoundException(__('Invalid server'));
}
}
$s = $this->Server->find('first', [
'conditions' => ['id' => $id],
'recursive' => -1,
]);
if (empty($s)) {
throw new NotFoundException(__('Invalid server'));
}
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$s = $this->Server->read(null, $id);
$error = false;
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
if (false == $this->Server->data['Server']['pull'] && ($technique == 'full' || $technique == 'incremental')) {
if (false == $s['Server']['pull'] && ($technique === 'full' || $technique === 'incremental')) {
$error = __('Pull setting not enabled for this server.');
}
if (false == $this->Server->data['Server']['pull_galaxy_clusters'] && ($technique == 'pull_relevant_clusters')) {
if (false == $s['Server']['pull_galaxy_clusters'] && ($technique === 'pull_relevant_clusters')) {
$error = __('Pull setting not enabled for this server.');
}
if (empty($error)) {
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
$result = $this->Server->pull($this->Auth->user(), $technique, $s);
if (is_array($result)) {
$success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
} else {
@ -741,24 +753,28 @@ class ServersController extends AppController
$this->set('pulledSightings', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'pull',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Pulling.'),
$jobId = $this->Job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'pull',
'Server: ' . $id,
__('Pulling.')
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
$this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'pull',
$this->Auth->user('id'),
$id,
$technique,
$jobId
],
false,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$success = __('Pull queued for background execution. Job ID: %s', $jobId);
}
}
@ -824,25 +840,30 @@ class ServersController extends AppController
}
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'push',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Pushing.'),
$jobId = $this->Job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'push',
'Server: ' . $id,
__('Pushing.')
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('push', $this->Auth->user('id'), $id, $jobId)
$this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'push',
$this->Auth->user('id'),
$id,
$technique,
$jobId
],
false,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$message = sprintf(__('Push queued for background execution. Job ID: %s'), $jobId);
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'push', $message, $this->response->type());
}
@ -946,8 +967,10 @@ class ServersController extends AppController
'Encryption' => array('count' => 0, 'errors' => 0, 'severity' => 5),
'Proxy' => array('count' => 0, 'errors' => 0, 'severity' => 5),
'Security' => array('count' => 0, 'errors' => 0, 'severity' => 5),
'Plugin' => array('count' => 0, 'errors' => 0, 'severity' => 5)
'Plugin' => array('count' => 0, 'errors' => 0, 'severity' => 5),
'SimpleBackgroundJobs' => array('count' => 0, 'errors' => 0, 'severity' => 5)
);
$writeableErrors = array(0 => __('OK'), 1 => __('not found'), 2 => __('is not writeable'));
$readableErrors = array(0 => __('OK'), 1 => __('not readable'));
$gpgErrors = array(0 => __('OK'), 1 => __('FAIL: settings not set'), 2 => __('FAIL: Failed to load GnuPG'), 3 => __('FAIL: Issues with the key/passphrase'), 4 => __('FAIL: sign failed'));
@ -955,6 +978,13 @@ class ServersController extends AppController
$zmqErrors = array(0 => __('OK'), 1 => __('not enabled (so not tested)'), 2 => __('Python ZeroMQ library not installed correctly.'), 3 => __('ZeroMQ script not running.'));
$sessionErrors = array(0 => __('OK'), 1 => __('High'), 2 => __('Alternative setting used'), 3 => __('Test failed'));
$moduleErrors = array(0 => __('OK'), 1 => __('System not enabled'), 2 => __('No modules found'));
$backgroundJobsErrors = array(
0 => __('OK'),
1 => __('Not configured (so not tested)'),
2 => __('Error connecting to Redis.'),
3 => __('Error connecting to Supervisor.'),
4 => __('Error connecting to Redis and Supervisor.')
);
$finalSettings = $this->Server->serverSettingsRead();
$issues = array(
@ -1020,20 +1050,19 @@ class ServersController extends AppController
$attachmentTool = new AttachmentTool();
try {
$advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
$advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus();
} catch (Exception $e) {
$this->log($e->getMessage(), LOG_NOTICE);
$advanced_attachments = false;
}
$this->set('advanced_attachments', $advanced_attachments);
// check if the current version of MISP is outdated or not
$version = $this->__checkVersion();
$this->set('version', $version);
$gitStatus = $this->Server->getCurrentGitStatus();
$gitStatus = $this->Server->getCurrentGitStatus(true);
$this->set('branch', $gitStatus['branch']);
$this->set('commit', $gitStatus['commit']);
$this->set('latestCommit', $gitStatus['latestCommit']);
$this->set('version', $gitStatus['version']);
$phpSettings = array(
'max_execution_time' => array(
@ -1067,7 +1096,7 @@ class ServersController extends AppController
}
$this->set('phpSettings', $phpSettings);
if ($version && (!$version['upToDate'] || $version['upToDate'] == 'older')) {
if ($gitStatus['version'] && $gitStatus['version']['upToDate'] === 'older') {
$diagnostic_errors++;
}
@ -1085,6 +1114,9 @@ class ServersController extends AppController
// if Proxy is set up in the settings, try to connect to a test URL
$proxyStatus = $this->Server->proxyDiagnostics($diagnostic_errors);
// if SimpleBackgroundJobs is set up in the settings, try to connect to Redis
$backgroundJobsStatus = $this->Server->backgroundJobsDiagnostics($diagnostic_errors);
// get the DB diagnostics
$dbDiagnostics = $this->Server->dbSpaceUsage();
$dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic();
@ -1139,7 +1171,7 @@ class ServersController extends AppController
unset($dumpResults[$key]['description']);
}
$dump = array(
'version' => $version,
'version' => $gitStatus['version'],
'phpSettings' => $phpSettings,
'gpgStatus' => $gpgErrors[$gpgStatus['status']],
'proxyStatus' => $proxyErrors[$proxyStatus],
@ -1154,7 +1186,8 @@ class ServersController extends AppController
'redisInfo' => $redisInfo,
'finalSettings' => $dumpResults,
'extensions' => $extensions,
'workers' => $worker_array
'workers' => $worker_array,
'backgroundJobsStatus' => $backgroundJobsErrors[$backgroundJobsStatus]
);
foreach ($dump['finalSettings'] as $k => $v) {
if (!empty($v['redacted'])) {
@ -1178,7 +1211,6 @@ class ServersController extends AppController
$this->set('phptoonew', $this->phptoonew);
$this->set('pythonmin', $this->pythonmin);
$this->set('pythonrec', $this->pythonrec);
$this->set('pymisp', $this->pymisp);
$this->set('title_for_layout', __('Diagnostics'));
}
@ -1187,10 +1219,25 @@ class ServersController extends AppController
if (!$this->request->is('post')) {
throw new MethodNotAllowedException();
}
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$message = __('Worker start signal sent');
$this->Server->getBackgroundJobsTool()->startWorkerByQueue($type);
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'startWorker', $type, $this->response->type(), $message);
} else {
$this->Flash->info($message);
$this->redirect('/servers/serverSettings/workers');
}
}
// CakeResque
$validTypes = array('default', 'email', 'scheduler', 'cache', 'prio', 'update');
if (!in_array($type, $validTypes)) {
throw new MethodNotAllowedException('Invalid worker type.');
}
$prepend = '';
if ($type != 'scheduler') {
$workerIssueCount = 0;
@ -1222,8 +1269,21 @@ class ServersController extends AppController
if (!$this->request->is('post')) {
throw new MethodNotAllowedException();
}
$this->Server->killWorker($pid, $this->Auth->user());
$message = __('Worker stop signal sent');
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->stopWorker($pid);
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->type(), $message);
} else {
$this->Flash->info($message);
$this->redirect('/servers/serverSettings/workers');
}
}
// CakeResque
$this->Server->killWorker($pid, $this->Auth->user());
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->type(), $message);
} else {
@ -1243,32 +1303,6 @@ class ServersController extends AppController
return $this->RestResponse->viewData($worker_array);
}
private function __checkVersion()
{
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
try {
$HttpSocket = $syncTool->setupHttpSocket();
$response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags');
$tags = $response->body;
} catch (Exception $e) {
return false;
}
if ($response->isOK() && !empty($tags)) {
$json_decoded_tags = json_decode($tags);
// find the latest version tag in the v[major].[minor].[hotfix] format
for ($i = 0; $i < count($json_decoded_tags); $i++) {
if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $json_decoded_tags[$i]->name)) {
break;
}
}
return $this->Server->checkVersion($json_decoded_tags[$i]->name);
} else {
return false;
}
}
public function idTranslator($localId = null)
{
// We retrieve the list of remote servers that we can query
@ -1436,18 +1470,18 @@ class ServersController extends AppController
}
}
$this->autoRender = false;
$this->loadModel('Log');
if (!is_writeable(APP . 'Config/config.php')) {
if (!Configure::read('MISP.system_setting_db') && !is_writeable(APP . 'Config/config.php')) {
$this->loadModel('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $this->Auth->user('Organisation')['name'],
'model' => 'Server',
'model_id' => 0,
'email' => $this->Auth->user('email'),
'action' => 'serverSettingsEdit',
'user_id' => $this->Auth->user('id'),
'title' => 'Server setting issue',
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $this->request->data['Server']['value'] . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.',
'org' => $this->Auth->user('Organisation')['name'],
'model' => 'Server',
'model_id' => 0,
'email' => $this->Auth->user('email'),
'action' => 'serverSettingsEdit',
'user_id' => $this->Auth->user('id'),
'title' => 'Server setting issue',
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $this->request->data['Server']['value'] . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.',
));
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'app/Config.config.php is not writeable to the apache user.', $this->response->type());
@ -1489,7 +1523,14 @@ class ServersController extends AppController
if (!$this->request->is('post')) {
throw new MethodNotAllowedException();
}
$this->Server->restartWorkers($this->Auth->user());
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->restartWorkers();
} else {
// CakeResque
$this->Server->restartWorkers($this->Auth->user());
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Server', 'restartWorkers', false, $this->response->type(), __('Restarting workers.'));
}
@ -1501,7 +1542,14 @@ class ServersController extends AppController
if (!$this->request->is('post')) {
throw new MethodNotAllowedException();
}
$this->Server->restartDeadWorkers($this->Auth->user());
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->restartDeadWorkers();
} else {
// CakeResque
$this->Server->restartDeadWorkers($this->Auth->user());
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Server', 'restartDeadWorkers', false, $this->response->type(), __('Restarting workers.'));
}
@ -1708,8 +1756,8 @@ class ServersController extends AppController
if (!empty($result)) {
$this->set('events', $result['publishCount']);
$this->set('messages', $result['messageCount']);
$this->set('time', date('Y/m/d H:i:s', $result['timestamp']));
$this->set('time2', date('Y/m/d H:i:s', $result['timestampSettings']));
$this->set('time', $result['timestamp']);
$this->set('time2', $result['timestampSettings']);
}
$this->render('ajax/zeromqstatus');
}
@ -1727,12 +1775,19 @@ class ServersController extends AppController
if (!$this->request->is('Post') || $this->request->is('ajax')) {
throw new MethodNotAllowedException();
}
$worker_array = array('cache', 'default', 'email', 'prio');
if (!in_array($worker, $worker_array)) {
throw new MethodNotAllowedException('Invalid worker');
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->purgeQueue($worker);
} else {
// CakeResque
$worker_array = array('cache', 'default', 'email', 'prio');
if (!in_array($worker, $worker_array)) {
throw new MethodNotAllowedException('Invalid worker');
}
$redis = Resque::redis();
$redis->del('queue:' . $worker);
}
$redis = Resque::redis();
$redis->del('queue:' . $worker);
$this->Flash->success('Queue cleared.');
$this->redirect($this->referer());
}
@ -1769,12 +1824,11 @@ class ServersController extends AppController
public function update($branch = false)
{
if ($this->request->is('post')) {
$branch = false;
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => ['branch'],
'ordered_url_params' => @compact($paramArray),
'ordered_url_params' => [],
'additional_delimiters' => PHP_EOL
);
$exception = false;
@ -1797,8 +1851,8 @@ class ServersController extends AppController
return new CakeResponse(array('body' => $update, 'type' => 'txt'));
}
} else {
$branch = $this->Server->getCurrentBranch();
$this->set('branch', $branch);
$this->set('isUpdatePossible', $this->Server->isUpdatePossible());
$this->set('branch', $this->Server->getCurrentBranch());
$this->render('ajax/update');
}
}
@ -2172,26 +2226,29 @@ misp.direct_call(relative_path, body)
public function cache($id = 'all')
{
if (Configure::read('MISP.background_jobs')) {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'cache_servers',
'job_input' => intval($id) ? $id : 'all',
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Starting server caching.'),
$jobId = $this->Job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'cache_servers',
intval($id) ? $id : 'all',
__('Starting server caching.')
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('cacheServer', $this->Auth->user('id'), $id, $jobId),
true
$this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'cacheServer',
$this->Auth->user('id'),
$id,
$jobId
],
false,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$message = 'Server caching job initiated.';
} else {
$result = $this->Server->cacheServerInitiator($this->Auth->user(), $id);
@ -2438,8 +2495,8 @@ misp.direct_call(relative_path, body)
public function removeOrphanedCorrelations()
{
$success = $this->Server->removeOrphanedCorrelations();
$message = __('Orphaned correlation removed');
$count = $this->Server->removeOrphanedCorrelations();
$message = __('%s orphaned correlation removed', $count);
if ($this->_isRest()) {
return $this->RestResponse->viewData($message, $this->response->type());
} else {
@ -2553,26 +2610,28 @@ misp.direct_call(relative_path, body)
$this->Flash->success('Done. For more details check the audit logs.');
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
} else {
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'default',
'job_type' => 'upgrade_24',
'job_input' => 'Old database',
'status' => 0,
'retries' => 0,
'org_id' => 0,
'message' => 'Job created.',
$this->loadModel('Job');
$jobId = $this->Job->createJob(
$this->Auth->user(),
Job::WORKER_DEFAULT,
'upgrade_24',
'Old database',
__('Job created.')
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('jobUpgrade24', $jobId, $this->Auth->user('id')),
true
$this->Server->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobUpgrade24',
$jobId,
$this->Auth->user('id'),
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).'));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
}

36
app/Controller/ShadowAttributesController.php
View File

@ -708,7 +708,7 @@ class ShadowAttributesController extends AppController
}
throw new InternalErrorException(__('Could not save the proposal. Errors: %s', $message));
} else {
$this->Flash->error(__('The ShadowAttribute could not be saved. Please, try again.'));
$this->Flash->error(__('The proposed Attribute could not be saved. Please, try again.'));
}
}
} else {
@ -1074,25 +1074,27 @@ class ShadowAttributesController extends AppController
$this->Flash->success(__('All done. ' . $k . ' proposals processed.'));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
} else {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'default',
'job_type' => 'generate proposal correlation',
'job_input' => 'All attributes',
'status' => 0,
'retries' => 0,
'org' => 'ADMIN',
'message' => 'Job created.',
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate proposal correlation',
'All attributes',
'Job created.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('jobGenerateShadowAttributeCorrelation', $jobId)
$this->Attribute->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateShadowAttributeCorrelation',
$jobId
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).'));
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
}

33
app/Controller/SightingsController.php
View File

@ -9,11 +9,6 @@ class SightingsController extends AppController
{
public $components = array('Session', 'RequestHandler');
public function beforeFilter()
{
parent::beforeFilter();
}
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
@ -81,7 +76,7 @@ class SightingsController extends AppController
$error_message = 'Could not add the Sighting. Reason: ' . $error;
return new CakeResponse(array('body' => json_encode(array('saved' => false, 'errors' => $error_message)), 'status' => 200, 'type' => 'json'));
} else {
return new CakeResponse(array('body' => json_encode(array('saved' => true, 'success' => $result . ' ' . $this->Sighting->type[$type] . (($result == 1) ? '' : 's') . ' added.')), 'status' => 200, 'type' => 'json'));
return new CakeResponse(array('body' => json_encode(array('saved' => true, 'success' => $result . ' ' . Sighting::TYPE[$type] . (($result == 1) ? '' : 's') . ' added.')), 'status' => 200, 'type' => 'json'));
}
} else {
if ($error) {
@ -137,8 +132,7 @@ class SightingsController extends AppController
throw new MethodNotAllowedException('Invalid attribute.');
}
} else {
$this->loadModel('Event');
$events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true));
$events = $this->Sighting->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true));
if (empty($events)) {
throw new MethodNotAllowedException('Invalid event.');
}
@ -254,24 +248,16 @@ class SightingsController extends AppController
public function index($eventid = false)
{
$this->loadModel('Event');
$sightingConditions = array();
if ($eventid) {
$sightingConditions = array('Sighting.event_id' => $eventid);
}
$sightedEvents = $this->Sighting->find('list', array(
'group' => ['Sighting.id', 'Sighting.event_id'],
$sightingConditions = $eventid ? array('Sighting.event_id' => $eventid) : [];
$sightedEvents = $this->Sighting->find('column', array(
'fields' => array('Sighting.event_id'),
'conditions' => $sightingConditions
'conditions' => $sightingConditions,
'unique' => true,
));
if (empty($sightedEvents)) {
$this->RestResponse->viewData(array());
}
$conditions = array('metadata' => true, 'contain' => false);
if ($eventid) {
$conditions['eventid'] = $sightedEvents;
}
$events = $this->Event->fetchEventIds($this->Auth->user(), [
$events = $this->Sighting->Event->fetchEventIds($this->Auth->user(), [
'eventIdList' => $sightedEvents
]);
$sightings = array();
@ -312,10 +298,9 @@ class SightingsController extends AppController
public function viewSightings($id, $context = 'attribute')
{
$this->loadModel('Event');
$id = $this->Sighting->explodeIdList($id);
if ($context === 'attribute') {
$objects = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1));
$objects = $this->Sighting->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1));
if (empty($objects)) {
throw new MethodNotAllowedException('Invalid object.');
}
@ -323,7 +308,7 @@ class SightingsController extends AppController
} elseif ($context === 'event') {
// let's set the context to event here, since we reuse the variable later on for some additional lookups.
// Passing $context = 'org' could have interesting results otherwise...
$events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]);
$events = $this->Sighting->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]);
$statistics = $this->Sighting->eventsStatistic($events, $this->Auth->user(), true);
} else {
throw new MethodNotAllowedException('Invalid context');

79
app/Controller/TagsController.php
View File

@ -505,6 +505,7 @@ class TagsController extends AppController
$this->loadModel('Taxonomy');
$expanded = array();
$this->set('taxonomy_id', $taxonomy_id);
$local_tag = !empty($this->params['named']['local']);
if ($taxonomy_id === 'collections') {
$this->loadModel('TagCollection');
// This method removes banned and hidden tags
@ -528,7 +529,7 @@ class TagsController extends AppController
}
} else {
if ($taxonomy_id === '0') {
$temp = $this->Taxonomy->getAllTaxonomyTags(true, $this->Auth->user(), true);
$temp = $this->Taxonomy->getAllTaxonomyTags(true, $this->Auth->user(), true, true, $local_tag);
$tags = array();
foreach ($temp as $tag) {
$tags[$tag['Tag']['id']] = $tag['Tag'];
@ -543,6 +544,9 @@ class TagsController extends AppController
'Tag.user_id' => array(0, $this->Auth->user('id')),
'Tag.hide_tag' => 0,
);
if (!$local_tag) {
$conditions['Tag.local_only'] = 0;
}
$favTags = $this->Tag->FavouriteTag->find('all', array(
'conditions' => $conditions,
'recursive' => -1,
@ -562,6 +566,9 @@ class TagsController extends AppController
$conditions['Tag.org_id'] = array(0, $this->Auth->user('org_id'));
$conditions['Tag.user_id'] = array(0, $this->Auth->user('id'));
}
if (!$local_tag) {
$conditions['Tag.local_only'] = 0;
}
$allTags = $this->Tag->find('all', array(
'conditions' => $conditions,
'recursive' => -1,
@ -585,6 +592,9 @@ class TagsController extends AppController
if ($tag['hide_tag']) {
continue; // do not include hidden tags
}
if ($tag['local_only'] && !$local_tag) {
continue; // we skip the local tags for global entries
}
if (!$isSiteAdmin) {
// Skip all tags that this user cannot use for tagging, determined by the org restriction on tags
if ($tag['org_id'] != '0' && $tag['org_id'] != $this->Auth->user('org_id')) {
@ -706,19 +716,19 @@ class TagsController extends AppController
return $this->RestResponse->viewData($results, 'json');
}
/**
* @param string $object_uuid Attribute or Event UUID
* @param string $type
* @param string $scope
* @return array
* @throws MethodNotAllowedException
*/
private function __findObjectByUuid($object_uuid, &$type, $scope = 'modify')
{
$this->loadModel('Event');
if (!$this->userRole['perm_tagger']) {
throw new MethodNotAllowedException(__('This functionality requires tagging permission.'));
}
$object = $this->Event->fetchEvent($this->Auth->user(), array(
'event_uuid' => $object_uuid,
'metadata' => 1
));
$type = 'Event';
$object = $this->Event->fetchSimpleEvent($this->Auth->user(), $object_uuid);
if (!empty($object)) {
$object = $object[0];
$type = 'Event';
if (
$scope !== 'view' &&
!$this->_isSiteAdmin() &&
@ -732,17 +742,12 @@ class TagsController extends AppController
}
} else {
$type = 'Attribute';
$object = $this->Event->Attribute->fetchAttributes(
$this->Auth->user(),
array(
'conditions' => array(
'Attribute.uuid' => $object_uuid
),
'flatten' => 1
)
);
$object = $this->Event->Attribute->fetchAttributeSimple($this->Auth->user(), [
'conditions' => array(
'Attribute.uuid' => $object_uuid
),
]);
if (!empty($object)) {
$object = $object[0];
if (
$scope !== 'view' &&
!$this->_isSiteAdmin() &&
@ -789,7 +794,7 @@ class TagsController extends AppController
$successes = 0;
$fails = array();
$existingRelations = array();
foreach ($tags as $k => $tag) {
foreach ($tags as $tag) {
if (is_numeric($tag)) {
$conditions = array('Tag.id' => $tag);
} else {
@ -813,13 +818,12 @@ class TagsController extends AppController
$fails[] = __('Tag not found and insufficient privileges to create it.');
continue;
}
$this->Tag->create();
$result = $this->Tag->save(array('Tag' => array('name' => $tag, 'colour' => $this->Tag->random_color())));
if (!$result) {
$createdTagId = $this->Tag->quickAdd($tag);
if (!$createdTagId) {
$fails[] = __('Unable to create tag. Reason: ' . json_encode($this->Tag->validationErrors));
continue;
}
$existingTag = $this->Tag->find('first', array('recursive' => -1, 'conditions' => array('Tag.id' => $this->Tag->id)));
$existingTag = $this->Tag->find('first', array('recursive' => -1, 'conditions' => array('Tag.id' => $createdTagId)));
} else {
$fails[] = __('Invalid Tag.');
continue;
@ -835,30 +839,30 @@ class TagsController extends AppController
continue;
}
}
if ($existingTag['Tag']['local_only'] && !$local) {
$fails[] = __('Invalid Tag. This tag can only be set as a local tag.');
continue;
}
$this->loadModel($objectType);
$connectorObject = $objectType . 'Tag';
$conditions = array(
strtolower($objectType) . '_id' => $object[$objectType]['id'],
'tag_id' => $existingTag['Tag']['id'],
'local' => ($local ? 1 : 0)
);
$existingAssociation = $this->$objectType->$connectorObject->find('first', array(
'conditions' => $conditions
));
if (!empty($existingAssociation)) {
$existingAssociation = $this->$objectType->$connectorObject->hasAny($conditions);
if ($existingAssociation) {
$message = __('%s already has the requested tag attached, no changes had to be made for tag %s.', $objectType, $existingTag['Tag']['name']);
$existingRelations[] = $existingTag['Tag']['name'];
$successes++;
continue;
}
$this->$objectType->$connectorObject->create();
$data = array(
$connectorObject => $conditions
);
if ($objectType == 'Attribute') {
$data[$connectorObject]['event_id'] = $object['Event']['id'];
$data = $conditions;
$data['local'] = $local ? 1 : 0;
if ($objectType === 'Attribute') {
$data['event_id'] = $object['Event']['id'];
}
$result = $this->$objectType->$connectorObject->save($data);
$result = $this->$objectType->$connectorObject->save([$connectorObject => $data]);
if ($result) {
if ($local) {
$message = 'Local tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
@ -867,8 +871,7 @@ class TagsController extends AppController
'recursive' => -1,
'conditions' => array($objectType . '.id' => $object[$objectType]['id'])
));
$date = new DateTime();
$tempObject[$objectType]['timestamp'] = $date->getTimestamp();
$tempObject[$objectType]['timestamp'] = time();
$this->$objectType->save($tempObject);
if ($objectType === 'Attribute') {
$this->$objectType->Event->unpublishEvent($object['Event']['id']);

121
app/Controller/UserSettingsController.php
View File

@ -114,8 +114,8 @@ class UserSettingsController extends AppController
$this->paginate['conditions'] = $conditions;
$data = $this->paginate();
foreach ($data as $k => $v) {
if (!empty($this->UserSetting->validSettings[$v['UserSetting']['setting']])) {
$data[$k]['UserSetting']['restricted'] = empty($this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']) ? '' : $this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted'];
if (!empty(UserSetting::VALID_SETTINGS[$v['UserSetting']['setting']])) {
$data[$k]['UserSetting']['restricted'] = empty(UserSetting::VALID_SETTINGS[$v['UserSetting']['setting']]['restricted']) ? '' : UserSetting::VALID_SETTINGS[$v['UserSetting']['setting']]['restricted'];
} else {
$data[$k]['UserSetting']['restricted'] = array();
}
@ -215,7 +215,6 @@ class UserSettingsController extends AppController
return $this->RestResponse->describe('UserSettings', 'setSetting', false, $this->response->type());
} else {
// load the valid settings from the model
$validSettings = $this->UserSetting->validSettings;
if ($this->_isSiteAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'recursive' => -1,
@ -235,33 +234,46 @@ class UserSettingsController extends AppController
}
$this->set('setting', $setting);
$this->set('users', $users);
$this->set('validSettings', $validSettings);
$this->set('validSettings', UserSetting::VALID_SETTINGS);
}
}
public function getSetting($user_id, $setting)
public function getSetting($userId = null, $setting = null)
{
if (!$this->UserSetting->checkSettingValidity($setting)) {
throw new MethodNotAllowedException(__('Invalid setting.'));
if ($this->request->is('post')) {
if (empty($this->request->data['setting'])) {
throw new BadRequestException("No setting name provided.");
}
$setting = $this->request->data['setting'];
$userId = $this->request->data['user_id'] ?? $this->Auth->user('id');
} else {
if (empty($userId) || empty($setting)) {
throw new BadRequestException("No setting name or user ID provided.");
}
}
if (!$this->UserSetting->checkSettingValidity($setting)) {
throw new NotFoundException(__('Invalid setting.'));
}
$userSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.user_id' => $user_id,
'UserSetting.setting' => $setting
),
'conditions' => [
'UserSetting.user_id' => $userId,
'UserSetting.setting' => $setting,
],
'contain' => array('User.id', 'User.org_id')
));
$checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting, $user_id);
if (empty($checkAccess)) {
throw new MethodNotAllowedException(__('Invalid setting.'));
if (empty($userSetting)) {
throw new NotFoundException(__('Invalid setting.'));
}
if (!empty($userSetting)) {
$userSetting = json_encode($userSetting['UserSetting']['value']);
} else {
$userSetting = '[]';
$checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting, $userId);
if (!$checkAccess) {
throw new NotFoundException(__('Invalid setting.'));
}
return $this->RestResponse->viewData($userSetting, $this->response->type(), false, true);
return $this->RestResponse->viewData($userSetting['UserSetting'], $this->response->type());
}
public function delete($id = false)
@ -274,15 +286,30 @@ class UserSettingsController extends AppController
*/
return $this->RestResponse->describe('UserSettings', 'delete', false, $this->response->type());
}
// check if the ID is valid and whether a user setting with the given ID exists
if (empty($id) || !is_numeric($id)) {
throw new InvalidArgumentException(__('Invalid ID passed.'));
if (!$this->request->is('post') && !$this->request->is('delete')) {
throw new MethodNotAllowedException(__('Expecting POST or DELETE request.'));
}
if (empty($id)) {
if (empty($this->request->data['setting'])) {
throw new BadRequestException("No setting name to delete provided.");
}
$conditions = ['UserSetting.setting' => $this->request->data['setting']];
if (!empty($this->request->data['user_id'])) {
$conditions['UserSetting.user_id'] = $this->request->data['user_id'];
} else {
$conditions['UserSetting.user_id'] = $this->Auth->user('id'); // current user
}
} else if (is_numeric($id)) {
$conditions = ['UserSetting.id' => $id];
} else {
throw new BadRequestException(__('Invalid ID passed.'));
}
$userSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.id' => $id
),
'conditions' => $conditions,
'contain' => array('User.id', 'User.org_id')
));
if (empty($userSetting)) {
@ -296,34 +323,30 @@ class UserSettingsController extends AppController
if ($settingPermCheck !== true) {
throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
}
if ($this->request->is('post') || $this->request->is('delete')) {
// Delete the setting that we were after.
$result = $this->UserSetting->delete($userSetting['UserSetting']['id']);
if ($result) {
// set the response for both the UI and API
$message = __('Setting deleted.');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('UserSettings', 'delete', $id, $this->response->type(), $message);
} else {
$this->Flash->success($message);
}
// Delete the setting that we were after.
$result = $this->UserSetting->delete($userSetting['UserSetting']['id']);
if ($result) {
// set the response for both the UI and API
$message = __('Setting deleted.');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('UserSettings', 'delete', $userSetting['UserSetting']['id'], $this->response->type(), $message);
} else {
// set the response for both the UI and API
$message = __('Setting could not be deleted.');
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('UserSettings', 'delete', $id, $message, $this->response->type());
} else {
$this->Flash->error($message);
}
$this->Flash->success($message);
}
/*
* The API responses stopped executing this function and returned a serialised response to the user.
* For UI users, redirect to where they issued the request from.
*/
$this->redirect($this->referer());
} else {
throw new MethodNotAllowedException(__('Expecting POST or DELETE request.'));
// set the response for both the UI and API
$message = __('Setting could not be deleted.');
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('UserSettings', 'delete', $userSetting['UserSetting']['id'], $message, $this->response->type());
} else {
$this->Flash->error($message);
}
}
/*
* The API responses stopped executing this function and returned a serialised response to the user.
* For UI users, redirect to where they issued the request from.
*/
$this->redirect($this->referer());
}
public function setHomePage()

35
app/Controller/UsersController.php
View File

@ -741,7 +741,7 @@ class UsersController extends AppController
$notification_message .= ' ' . __('User notification of new credentials could not be send.');
}
}
if (!empty(Configure::read('Security.advanced_authkeys'))) {
if (!empty(Configure::read('Security.advanced_authkeys')) && $this->_isRest()) {
$this->loadModel('AuthKey');
$newKey = $this->AuthKey->createnewkey($this->User->id);
}
@ -1210,8 +1210,7 @@ class UsersController extends AppController
}
}
// populate the DB with the first role (site admin) if it's empty
$this->loadModel('Role');
if ($this->Role->find('count') == 0) {
if (!$this->User->Role->hasAny()) {
$siteAdmin = array('Role' => array(
'id' => 1,
'name' => 'Site Admin',
@ -1230,14 +1229,14 @@ class UsersController extends AppController
'perm_template' => 1,
'perm_tagger' => 1,
));
$this->Role->save($siteAdmin);
$this->User->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource == 'Database/Postgres') {
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->Role->query($sql);
$this->User->Role->query($sql);
}
}
if ($this->User->Organisation->find('count', array('conditions' => array('Organisation.local' => true))) == 0) {
if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
$this->User->runUpdates();
$date = date('Y-m-d H:i:s');
$org = array('Organisation' => array(
@ -1253,23 +1252,25 @@ class UsersController extends AppController
));
$this->User->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource == 'Database/Postgres') {
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->User->Organisation->query($sql);
}
$org_id = $this->User->Organisation->id;
} else {
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$org_id = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$org_id = $firstOrg['Organisation']['id'];
}
}
// populate the DB with the first user if it's empty
if ($this->User->find('count') == 0) {
if (!$this->User->hasAny()) {
if (!isset($org_id)) {
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$org_id = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$org_id = $firstOrg['Organisation']['id'];
}
}
$this->User->runUpdates();
$this->User->createInitialUser($org_id);
}

2
app/Lib/Dashboard/MispStatusWidget.php
View File

@ -44,7 +44,7 @@ class MispStatusWidget
'View'
)
);
$notifications = $this->Event->populateNotifications($user);
$notifications = $this->Event->User->populateNotifications($user);
if (!empty($notifications['proposalCount'])) {
$data[] = array(
'title' => __('Pending proposals'),

64
app/Lib/Export/CsvExport.php
View File

@ -64,7 +64,7 @@ class CsvExport
$attribute['decay_score_score'] = 0;
$attribute['decay_score_decayed'] = false;
}
return $this->__addLine($attribute, $options);
return $this->__addLine($attribute);
}
private function __sightingsHandler($sighting, $options)
@ -89,7 +89,7 @@ class CsvExport
$sighting['Sighting'][$new_key] = $attribute_val;
}
}
$lines .= $this->__addLine($sighting['Sighting'], $options);
$lines .= $this->__addLine($sighting['Sighting']);
return $lines;
}
@ -97,20 +97,20 @@ class CsvExport
{
$lines = '';
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $k => $attribute) {
foreach ($event['Attribute'] as $attribute) {
$attribute = $this->__addMetadataToAttribute($event, $attribute);
$lines .= $this->__addLine($attribute, $options);
$lines .= $this->__addLine($attribute);
}
}
if (!empty($event['Object'])) {
foreach ($event['Object'] as $k => $object) {
foreach ($event['Object'] as $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
$attribute = $this->__addMetadataToAttribute($event, $attribute);
$attribute['object_uuid'] = $object['uuid'];
$attribute['object_name'] = $object['name'];
$attribute['object_meta-category'] = $object['meta-category'];
$lines .= $this->__addLine($attribute, $options);
$lines .= $this->__addLine($attribute);
}
}
}
@ -118,22 +118,27 @@ class CsvExport
return $lines;
}
private function __addLine($attribute, $options = array()) {
$line = '';
/**
* @param array $attribute
* @return string
*/
private function __addLine($attribute)
{
$parts = [];
foreach ($this->requested_fields as $req_att) {
if (empty($line)) {
$line = $this->__escapeCSVField($attribute[$req_att]);
} else {
$line .= ',' . $this->__escapeCSVField($attribute[$req_att]);
}
if (isset($attribute[$req_att])) {
$parts[] = $this->__escapeCSVField($attribute[$req_att]);
} else {
$parts[] = '""'; // keep it consistent with old CSV format
}
}
return $line . PHP_EOL;
return implode(',', $parts) . PHP_EOL;
}
private function __escapeCSVField(&$field)
private function __escapeCSVField($field)
{
if (is_bool($field)) {
return ($field ? '1' : '0');
return $field ? '1' : '0';
}
if (is_numeric($field)) {
return $field;
@ -257,15 +262,18 @@ class CsvExport
return '';
}
public function eventIndex($events)
/**
* @param array $events
* @return Generator[string]
*/
public function eventIndex(array $events)
{
$fields = array(
'id', 'date', 'info', 'tags', 'uuid', 'published', 'analysis', 'attribute_count', 'orgc_id', 'orgc_name', 'orgc_uuid', 'timestamp', 'distribution', 'sharing_group_id', 'threat_level_id',
'publish_timestamp', 'extends_uuid'
);
$result = implode(',', $fields) . PHP_EOL;
foreach ($events as $key => $event) {
$event['tags'] = '';
yield implode(',', $fields) . PHP_EOL;
foreach ($events as $event) {
if (!empty($event['EventTag'])) {
$tags = array();
foreach ($event['EventTag'] as $et) {
@ -275,16 +283,18 @@ class CsvExport
} else {
$tags = '';
}
$event['Event']['tags'] = $tags;
$event['Event']['orgc_name'] = $event['Orgc']['name'];
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
$event['tags'] = $tags;
$event['orgc_name'] = $event['Orgc']['name'];
$event['orgc_uuid'] = $event['Orgc']['uuid'];
$current = array();
foreach ($fields as $field) {
$current[] = $this->__escapeCSVField($event['Event'][$field]);
if (isset($event[$field])) {
$current[] = $this->__escapeCSVField($event[$field]);
} else {
$current[] = '';
}
}
$result .= implode(', ', $current) . PHP_EOL;
yield implode(',', $current) . PHP_EOL;
}
return $result;
}
}

1
app/Lib/Export/OpendataExport.php
View File

@ -3,7 +3,6 @@
class OpendataExport
{
public $non_restrictive_export = true;
public $use_default_filters = true;
public $mock_query_only = true;
private $__default_filters = null;

54
app/Lib/Export/Stix1Export.php
View File

@ -1,5 +1,4 @@
<?php
App::uses('StixExport', 'Export');
class Stix1Export extends StixExport
@ -7,42 +6,33 @@ class Stix1Export extends StixExport
protected $__attributes_limit = 15000;
protected $__default_version = '1.1.1';
protected $__sane_versions = array('1.1.1', '1.2');
private $__script_name = 'misp2stix.py ';
private $__baseurl = null;
private $__org = null;
protected function __initiate_framing_params()
{
$this->__baseurl = escapeshellarg(Configure::read('MISP.baseurl'));
$this->__org = escapeshellarg(Configure::read('MISP.org'));
$my_server = ClassRegistry::init('Server');
return $my_server->getPythonVersion() . ' ' . $this->__framing_script . ' stix1 -s ' . $this->__scope . ' -v ' . $this->__version . ' -n ' . $this->__baseurl . ' -o ' . $this->__org . ' -f ' . $this->__return_format . ' ' . $this->__end_of_cmd;
return [
ProcessTool::pythonBin(),
$this->__framing_script,
'stix1',
'-s', $this->__scope,
'-v', $this->__version,
'-n', Configure::read('MISP.baseurl'),
'-o', Configure::read('MISP.org'),
'-f', $this->__return_format,
];
}
protected function __parse_misp_data($filenames)
protected function __parse_misp_data()
{
$scriptFile = $this->__scripts_dir . $this->__script_name;
$my_server = ClassRegistry::init('Server');
$result = shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . '-v ' . $this->__version . ' -f ' . $this->__return_format . ' -o ' . $this->__org . ' -i ' . $this->__tmp_dir . $filenames . ' -s ' . $this->__scope . $this->__end_of_cmd);
$decoded = json_decode($result, true);
if (!isset($decoded['success']) || !$decoded['success']) {
$this->__delete_temporary_files();
$error = !empty($decoded['error']) ? $decoded['error'] : $result;
return 'Error while processing your query: ' . $error;
}
if (!empty($decoded['filenames'])) {
foreach ($this->__filenames as $f => $filename) {
@unlink($this->__tmp_dir . $filename);
}
foreach ($decoded['filenames'] as $filename) {
$this->__write_stix_content($filename);
}
} else {
foreach ($this->__filenames as $f => $filename) {
$content_filename = $this->__tmp_dir . $filename;
$this->__write_stix_content($content_filename . '.out');
@unlink($content_filename);
}
}
$command = [
ProcessTool::pythonBin(),
$this->__scripts_dir . 'misp2stix.py',
'-s', $this->__scope,
'-v', $this->__version,
'-f', $this->__return_format,
'-o', Configure::read('MISP.org'),
'-i',
];
$command = array_merge($command, $this->__filenames);
return = ProcessTool::execute($command, null, true);
}
}

38
app/Lib/Export/Stix2Export.php
View File

@ -1,5 +1,4 @@
<?php
App::uses('StixExport', 'Export');
class Stix2Export extends StixExport
@ -7,31 +6,30 @@ class Stix2Export extends StixExport
protected $__attributes_limit = 15000;
protected $__default_version = '2.0';
protected $__sane_versions = array('2.0', '2.1');
private $__script_name = 'stix2/misp2stix2.py ';
protected function __initiate_framing_params()
{
$my_server = ClassRegistry::init('Server');
return $my_server->getPythonVersion() . ' ' . $this->__framing_script . ' stix2 -v ' . $this->__version . ' --uuid ' . escapeshellarg(CakeText::uuid()) . $this->__end_of_cmd;
return [
ProcessTool::pythonBin(),
$this->__framing_script,
'stix2',
'-v', $this->__version,
'--uuid', CakeText::uuid(),
];
}
protected function __parse_misp_data($filenames)
protected function __parse_misp_data()
{
$scriptFile = $this->__scripts_dir . $this->__script_name;
$filenames = implode(' ' . $this->__tmp_dir, $this->__filenames);
$my_server = ClassRegistry::init('Server');
$result = shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . '-v ' . $this->__version . ' -i ' . $this->__tmp_dir . $filenames . $this->__end_of_cmd);
$scriptFile = $this->__scripts_dir . 'stix2/misp2stix2.py';
$command = [
ProcessTool::pythonBin(),
$scriptFile,
'-v', $this->__version,
'-i',
];
$command = array_merge($command, $this->__filenames);
$result = ProcessTool::execute($command, null, true);
$result = preg_split("/\r\n|\n|\r/", trim($result));
$decoded = json_decode(end($result), true);
if (!isset($decoded['success']) || !$decoded['success']) {
$this->__delete_temporary_files();
$error = !empty($decoded['error']) ? $decoded['error'] : $result;
return 'Error while processing your query: ' . $error;
}
foreach ($this->__filenames as $f => $filename) {
$content_filename = $this->__tmp_dir . $filename;
$this->__write_stix_content($content_filename . '.out');
@unlink($content_filename);
}
return end($result);
}
}

143
app/Lib/Export/StixExport.php
View File

@ -1,6 +1,10 @@
<?php
App::uses('JSONConverterTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
App::uses('JsonTool', 'Tools');
App::uses('ProcessTool', 'Tools');
class StixExport
abstract class StixExport
{
public $additional_params = array(
'includeEventTags' => 1,
@ -8,31 +12,31 @@ class StixExport
);
protected $__return_format = 'json';
protected $__scripts_dir = APP . 'files/scripts/';
protected $__tmp_dir = APP . 'files/scripts/tmp/';
protected $__framing_script = APP . 'files/scripts/misp_framing.py';
protected $__end_of_cmd = ' 2>' . APP . 'tmp/logs/exec-errors.log';
protected $__return_type = null;
/** @var array Full paths to files to convert */
protected $__filenames = array();
protected $__default_filters = null;
protected $__version = null;
protected $__scope = null;
protected $__stix_file = null;
protected $__framing = null;
protected $stixFile = null;
private $__cluster_uuids = array();
private $__converter = null;
private $__current_filename = null;
private $__empty_file = true;
private $__empty_file = null;
private $__event_galaxies = array();
/** @var File */
private $__tmp_file = null;
private $__n_attributes = 0;
public $non_restrictive_export = true;
public $use_default_filters = true;
private $Server;
public function setDefaultFilters($filters)
{
$sane_version = (!empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions));
$sane_version = !empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions, true);
$this->__version = $sane_version ? $filters['stix-version'] : $this->__default_version;
}
@ -63,25 +67,21 @@ class StixExport
public function header($options = array())
{
App::uses('JSONConverterTool', 'Tools');
$this->__converter = new JSONConverterTool();
$this->__scope = $options['scope'];
$this->__return_type = $options['returnFormat'];
if ($this->__return_type == 'stix-json') {
if ($this->__return_type === 'stix-json') {
$this->__return_type = 'stix';
} else if ($this->__return_type == 'stix') {
} else if ($this->__return_type === 'stix') {
$this->__return_format = 'xml';
}
$framing_cmd = $this->__initiate_framing_params();
$randomFileName = $this->__generateRandomFileName();
$this->__framing = json_decode(shell_exec($framing_cmd), true);
$this->__stix_file = new File($this->__tmp_dir . $randomFileName . '.' . $this->__return_type);
unset($randomFileName);
$this->__stix_file->write($this->__framing['header']);
$this->__initialize_misp_file();
return '';
}
/**
* @return TmpFileTool
* @throws Exception
*/
public function footer()
{
if ($this->__empty_file) {
@ -95,16 +95,33 @@ class StixExport
$this->__tmp_file->close();
$this->__filenames[] = $this->__current_filename;
}
$filenames = implode(' ' . $this->__tmp_dir, $this->__filenames);
$this->__parse_misp_data($filenames);
$stix_event = $this->__stix_file->read();
$this->__stix_file->close();
$this->__stix_file->delete();
$sep_len = strlen($this->__framing['separator']);
if ($sep_len != 0 && !empty($this->__filenames)) {
$stix_event = substr($stix_event, 0, -$sep_len);
$result = $this->__parse_misp_data();
$decoded = json_decode($result, true);
if (!isset($decoded['success']) || !$decoded['success']) {
if (!empty($decoded['filenames'])) {
$this->__delete_temporary_files(false, $decoded['filename']);
} else {
$this->__delete_temporary_files(true);
}
$error = $decoded && !empty($decoded['error']) ? $decoded['error'] : $result;
throw new Exception('Error while processing your query during STIX export: ' . $error);
}
return $stix_event . $this->__framing['footer'];
$this->__delete_temporary_files();
$framing = $this->getFraming();
$this->stixFile = new TmpFileTool();
$this->stixFile->write($framing['header']);
$separator = $framing['separator'];
if (!empty($decoded['filenames'])) {
foreach ($decoded['filenames'] as $filename) {
$this->__write_stix_content($filename, $separator);
}
} else {
foreach ($this->__filenames as $filename) {
$this->__write_stix_content($filename . '.out', $separator);
}
}
$this->stixFile->write($framing['footer']);
return $this->stixFile;
}
public function separator()
@ -251,22 +268,18 @@ class StixExport
$attributes_count += count($_object['Attribute']);
}
}
$event = $this->__converter->convert($event);
$converter = new JSONConverterTool();
$event = JsonTool::encode($converter->convert($data, false, true)); // we don't need pretty printed JSON
if ($this->__n_attributes + $attributes_count <= $this->__attributes_limit) {
$this->__tmp_file->append($this->__n_attributes == 0 ? $event : ', ' . $event);
$this->__n_attributes += $attributes_count;
$this->__empty_file = false;
} elseif ($attributes_count > $this->__attributes_limit) {
$filePath = FileAccessTool::writeToTempFile($event);
$this->__filenames[] = $filePath;
} else {
if ($attributes_count > $this->__attributes_limit) {
$randomFileName = $this->__generateRandomFileName();
$tmpFile = new File($this->__tmp_dir . $randomFileName, true, 0644);
$tmpFile->write($event);
$tmpFile->close();
$this->__filenames[] = $randomFileName;
} else {
$this->__terminate_misp_file($event);
$this->__n_attributes = $attributes_count;
}
$this->__terminate_misp_file($event);
$this->__n_attributes = $attributes_count;
}
return '';
}
@ -294,24 +307,31 @@ class StixExport
private function __initialize_misp_file()
{
$this->__current_filename = $this->__generateRandomFileName();
$this->__tmp_file = new File($this->__tmp_dir . $this->__current_filename, true, 0644);
$this->__current_filename = FileAccessTool::createTempFile();
$this->__tmp_file = new File($this->__current_filename);
$this->__tmp_file->write('{"response": ' . ($this->__scope === 'Attribute' ? '{"Attribute": [' : '['));
$this->__empty_file = true;
}
private function __generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}
protected function __delete_temporary_files()
{
foreach ($this->__filenames as $f => $filename) {
@unlink($this->__tmp_dir . $filename);
foreach ($this->__filenames as $filename) {
FileAccessTool::deleteFileIfExists($filename);
}
$this->__stix_file->close();
$this->__stix_file->delete();
}
/**
* @return array
* @throws Exception
*/
private function getFraming()
{
$framingCmd = $this->__initiate_framing_params();
$framing = json_decode(ProcessTool::execute($framingCmd, null, true), true);
if ($framing === null || isset($framing['error'])) {
throw new Exception("Could not get results from framing cmd when exporting STIX file.");
}
return $framing;
}
private function __merge_galaxy_tag(&$galaxies, $tag_name)
@ -345,13 +365,22 @@ class StixExport
$this->__event_galaxies = array();
}
protected function __write_stix_content($filename)
private function __write_stix_content($filename, $separator)
{
$file = new File($filename);
$stix_content = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1);
$file->close();
$file->delete();
$this->__stix_file->append($stix_content . $this->__framing['separator']);
unset($stix_content);
$stix_content = FileAccessTool::readAndDelete($filename);
if ($this->__return_type === 'stix') {
$stix_content = substr($stix_content, 1, -1);
}
$this->stixFile->writeWithSeparator($stix_content, $separator);
}
/**
* @return array
*/
abstract protected function __parse_misp_data();
/**
* @return array
*/
abstract protected function __initiate_framing_params();
}

69
app/Lib/Tools/AttachmentTool.php
View File

@ -1,5 +1,7 @@
<?php
App::uses('AWSS3Client', 'Tools');
App::uses('ProcessTool', 'Tools');
App::uses('JsonTool', 'Tools');
class AttachmentTool
{
@ -316,13 +318,13 @@ class AttachmentTool
'-j', // junk (don't record) directory names
'-P', // use standard encryption
self::ZIP_PASSWORD,
escapeshellarg($zipFile),
escapeshellarg($tempDir . DS . $md5),
escapeshellarg($tempDir . DS . $md5 . '.filename.txt'),
$zipFile,
$tempDir . DS . $md5,
$tempDir . DS . $md5 . '.filename.txt',
];
try {
$this->execute($exec);
ProcessTool::execute($exec);
return FileAccessTool::readFromFile($zipFile);
} catch (Exception $e) {
@ -360,13 +362,13 @@ class AttachmentTool
* @return array
* @throws Exception
*/
public function advancedExtraction($pythonBin, $filePath)
public function advancedExtraction($filePath)
{
return $this->executeAndParseJsonOutput([
$pythonBin,
ProcessTool::pythonBin(),
self::ADVANCED_EXTRACTION_SCRIPT_PATH,
'-p',
escapeshellarg($filePath),
$filePath,
]);
}
@ -375,9 +377,9 @@ class AttachmentTool
* @return array
* @throws Exception
*/
public function checkAdvancedExtractionStatus($pythonBin)
public function checkAdvancedExtractionStatus()
{
return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
return $this->executeAndParseJsonOutput([ProcessTool::pythonBin(), self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
}
/**
@ -496,50 +498,11 @@ class AttachmentTool
*/
private function executeAndParseJsonOutput(array $command)
{
$output = $this->execute($command);
$json = json_decode($output, true);
if ($json === null) {
throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
$output = ProcessTool::execute($command);
try {
return JsonTool::decode($output);
} catch (Exception $e) {
throw new Exception("Command output is not valid JSON.", 0, $e);
}
return $json;
}
/**
* This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
* can be much more specific.
*
* @param array $command
* @return string
* @throws Exception
*/
private function execute(array $command)
{
$descriptorspec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$command = implode(' ', $command);
$process = proc_open($command, $descriptorspec, $pipes);
if (!$process) {
throw new Exception("Command '$command' could be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
throw new Exception("Could not get STDOUT of command.");
}
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if ($returnCode !== 0) {
throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
}
return $stdout;
}
}

698
app/Lib/Tools/AttributeValidationTool.php Normal file
View File

@ -0,0 +1,698 @@
<?php
class AttributeValidationTool
{
// private
const HASH_HEX_LENGTH = [
'authentihash' => 64,
'md5' => 32,
'imphash' => 32,
'telfhash' => 70,
'sha1' => 40,
'git-commit-id' => 40,
'x509-fingerprint-md5' => 32,
'x509-fingerprint-sha1' => 40,
'x509-fingerprint-sha256' => 64,
'ja3-fingerprint-md5' => 32,
'jarm-fingerprint' => 62,
'hassh-md5' => 32,
'hasshserver-md5' => 32,
'pehash' => 40,
'sha224' => 56,
'sha256' => 64,
'sha384' => 96,
'sha512' => 128,
'sha512/224' => 56,
'sha512/256' => 64,
'sha3-224' => 56,
'sha3-256' => 64,
'sha3-384' => 96,
'sha3-512' => 128,
];
/**
* Do some last second modifications before the validation
* @param string $type
* @param mixed $value
* @return string
*/
public static function modifyBeforeValidation($type, $value)
{
$value = self::handle4ByteUnicode($value);
switch ($type) {
case 'ip-src':
case 'ip-dst':
return self::compressIpv6($value);
case 'md5':
case 'sha1':
case 'sha224':
case 'sha256':
case 'sha384':
case 'sha512':
case 'sha512/224':
case 'sha512/256':
case 'sha3-224':
case 'sha3-256':
case 'sha3-384':
case 'sha3-512':
case 'ja3-fingerprint-md5':
case 'jarm-fingerprint':
case 'hassh-md5':
case 'hasshserver-md5':
case 'hostname':
case 'pehash':
case 'authentihash':
case 'vhash':
case 'imphash':
case 'telfhash':
case 'tlsh':
case 'anonymised':
case 'cdhash':
case 'email':
case 'email-src':
case 'email-dst':
case 'target-email':
case 'whois-registrant-email':
return strtolower($value);
case 'domain':
$value = strtolower($value);
$value = trim($value, '.');
// Domain is not valid, try to convert to punycode
if (!self::isDomainValid($value) && function_exists('idn_to_ascii')) {
$punyCode = idn_to_ascii($value);
if ($punyCode !== false) {
$value = $punyCode;
}
}
return $value;
case 'domain|ip':
$value = strtolower($value);
$parts = explode('|', $value);
if (!isset($parts[1])) {
return $value; // not a composite
}
$parts[0] = trim($parts[0], '.');
// Domain is not valid, try to convert to punycode
if (!self::isDomainValid($parts[0]) && function_exists('idn_to_ascii')) {
$punyCode = idn_to_ascii($parts[0]);
if ($punyCode !== false) {
$parts[0] = $punyCode;
}
}
$parts[1] = self::compressIpv6($parts[1]);
return "$parts[0]|$parts[1]";
case 'filename|md5':
case 'filename|sha1':
case 'filename|imphash':
case 'filename|sha224':
case 'filename|sha256':
case 'filename|sha384':
case 'filename|sha512':
case 'filename|sha512/224':
case 'filename|sha512/256':
case 'filename|sha3-224':
case 'filename|sha3-256':
case 'filename|sha3-384':
case 'filename|sha3-512':
case 'filename|authentihash':
case 'filename|vhash':
case 'filename|pehash':
case 'filename|tlsh':
// Convert hash to lowercase
$pos = strpos($value, '|');
return substr($value, 0, $pos) . strtolower(substr($value, $pos));
case 'http-method':
case 'hex':
return strtoupper($value);
case 'vulnerability':
case 'weakness':
$value = str_replace('–', '-', $value);
return strtoupper($value);
case 'cc-number':
case 'bin':
return preg_replace('/[^0-9]+/', '', $value);
case 'iban':
case 'bic':
$value = strtoupper($value);
return preg_replace('/[^0-9A-Z]+/', '', $value);
case 'prtn':
case 'whois-registrant-phone':
case 'phone-number':
if (substr($value, 0, 2) == '00') {
$value = '+' . substr($value, 2);
}
$value = preg_replace('/\(0\)/', '', $value);
return preg_replace('/[^\+0-9]+/', '', $value);
case 'x509-fingerprint-md5':
case 'x509-fingerprint-sha256':
case 'x509-fingerprint-sha1':
$value = str_replace(':', '', $value);
return strtolower($value);
case 'ip-dst|port':
case 'ip-src|port':
if (substr_count($value, ':') >= 2) { // (ipv6|port) - tokenize ip and port
if (strpos($value, '|')) { // 2001:db8::1|80
$parts = explode('|', $value);
} elseif (strpos($value, '[') === 0 && strpos($value, ']') !== false) { // [2001:db8::1]:80
$ipv6 = substr($value, 1, strpos($value, ']')-1);
$port = explode(':', substr($value, strpos($value, ']')))[1];
$parts = array($ipv6, $port);
} elseif (strpos($value, '.')) { // 2001:db8::1.80
$parts = explode('.', $value);
} elseif (strpos($value, ' port ')) { // 2001:db8::1 port 80
$parts = explode(' port ', $value);
} elseif (strpos($value, 'p')) { // 2001:db8::1p80
$parts = explode('p', $value);
} elseif (strpos($value, '#')) { // 2001:db8::1#80
$parts = explode('#', $value);
} else { // 2001:db8::1:80 this one is ambiguous
$temp = explode(':', $value);
$parts = array(implode(':', array_slice($temp, 0, count($temp)-1)), end($temp));
}
} elseif (strpos($value, ':')) { // (ipv4:port)
$parts = explode(':', $value);
} elseif (strpos($value, '|')) { // (ipv4|port)
$parts = explode('|', $value);
} else {
return $value;
}
return self::compressIpv6($parts[0]) . '|' . $parts[1];
case 'mac-address':
case 'mac-eui-64':
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
return wordwrap($value, 2, ':', true);
case 'hostname|port':
$value = strtolower($value);
return str_replace(':', '|', $value);
case 'boolean':
$value = trim(strtolower($value));
if ('true' === $value) {
$value = 1;
} else if ('false' === $value) {
$value = 0;
}
return $value ? '1' : '0';
case 'datetime':
try {
return (new DateTime($value, new DateTimeZone('GMT')))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formatting with microseconds
} catch (Exception $e) {
return $value; // silently skip. Rejection will be done in validation()
}
case 'AS':
if (strtoupper(substr($value, 0, 2)) === 'AS') {
$value = substr($value, 2); // remove 'AS'
}
if (strpos($value, '.') !== false) { // maybe value is in asdot notation
$parts = explode('.', $value);
if (self::isPositiveInteger($parts[0]) && self::isPositiveInteger($parts[1])) {
return $parts[0] * 65536 + $parts[1];
}
}
return $value;
}
return $value;
}
/**
* Validate if value is valid for given attribute type.
* At this point, we can be sure, that composite type is really composite.
* @param string $type
* @param string $value
* @return bool|string
*/
public static function validate($type, $value)
{
switch ($type) {
case 'md5':
case 'imphash':
case 'telfhash':
case 'sha1':
case 'sha224':
case 'sha256':
case 'sha384':
case 'sha512':
case 'sha512/224':
case 'sha512/256':
case 'sha3-224':
case 'sha3-256':
case 'sha3-384':
case 'sha3-512':
case 'authentihash':
case 'ja3-fingerprint-md5':
case 'jarm-fingerprint':
case 'hassh-md5':
case 'hasshserver-md5':
case 'x509-fingerprint-md5':
case 'x509-fingerprint-sha256':
case 'x509-fingerprint-sha1':
case 'git-commit-id':
if (self::isHashValid($type, $value)) {
return true;
}
$length = self::HASH_HEX_LENGTH[$type];
return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
case 'tlsh':
if (self::isTlshValid($value)) {
return true;
}
return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
case 'pehash':
if (self::isHashValid('pehash', $value)) {
return true;
}
return __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
case 'ssdeep':
if (self::isSsdeep($value)) {
return true;
}
return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash');
case 'impfuzzy':
if (substr_count($value, ':') === 2) {
$parts = explode(':', $value);
if (self::isPositiveInteger($parts[0])) {
return true;
}
}
return __('Invalid impfuzzy format. The format has to be imports:hash:hash');
case 'cdhash':
if (preg_match("#^[0-9a-f]{40,}$#", $value)) {
return true;
}
return __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)');
case 'http-method':
if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) {
return true;
}
return __('Unknown HTTP method.');
case 'filename|pehash':
// no newline
if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) {
return true;
}
return __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
case 'filename|md5':
case 'filename|sha1':
case 'filename|imphash':
case 'filename|sha224':
case 'filename|sha256':
case 'filename|sha384':
case 'filename|sha512':
case 'filename|sha512/224':
case 'filename|sha512/256':
case 'filename|sha3-224':
case 'filename|sha3-256':
case 'filename|sha3-384':
case 'filename|sha3-512':
case 'filename|authentihash':
$hashType = substr($type, 9); // strip `filename|`
$length = self::HASH_HEX_LENGTH[$hashType];
if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) {
return true;
}
return __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length);
case 'filename|ssdeep':
$composite = explode('|', $value);
if (strpos($composite[0], "\n") !== false) {
return __('Filename must not contain new line character.');
}
if (self::isSsdeep($composite[1])) {
return true;
}
return __('Invalid ssdeep hash (expected: blocksize:hash:hash).');
case 'filename|tlsh':
$composite = explode('|', $value);
if (strpos($composite[0], "\n") !== false) {
return __('Filename must not contain new line character.');
}
if (self::isTlshValid($composite[1])) {
return true;
}
return __('TLSH hash has an invalid length or format (expected: filename|at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
case 'filename|vhash':
if (preg_match('#^.+\|.+$#', $value)) {
return true;
}
return __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
case 'ip-src':
case 'ip-dst':
if (strpos($value, '/') !== false) {
$parts = explode("/", $value);
if (count($parts) !== 2 || !self::isPositiveInteger($parts[1])) {
return __('Invalid CIDR notation value found.');
}
if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
if ($parts[1] > 32) {
return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.');
}
} else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
if ($parts[1] > 128) {
return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.');
}
} else {
return __('IP address has an invalid format.');
}
} else if (!filter_var($value, FILTER_VALIDATE_IP)) {
return __('IP address has an invalid format.');
}
return true;
case 'port':
if (!self::isPortValid($value)) {
return __('Port numbers have to be integers between 1 and 65535.');
}
return true;
case 'ip-dst|port':
case 'ip-src|port':
$parts = explode('|', $value);
if (!filter_var($parts[0], FILTER_VALIDATE_IP)) {
return __('IP address has an invalid format.');
}
if (!self::isPortValid($parts[1])) {
return __('Port numbers have to be integers between 1 and 65535.');
}
return true;
case 'mac-address':
return preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value) === 1;
case 'mac-eui-64':
return preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value) === 1;
case 'hostname':
case 'domain':
if (self::isDomainValid($value)) {
return true;
}
return __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
case 'hostname|port':
$parts = explode('|', $value);
if (!self::isDomainValid($parts[0])) {
return __('Hostname has an invalid format.');
}
if (!self::isPortValid($parts[1])) {
return __('Port numbers have to be integers between 1 and 65535.');
}
return true;
case 'domain|ip':
$parts = explode('|', $value);
if (!self::isDomainValid($parts[0])) {
return __('Domain has an invalid format.');
}
if (!filter_var($parts[1], FILTER_VALIDATE_IP)) {
return __('IP address has an invalid format.');
}
return true;
case 'email':
case 'email-src':
case 'eppn':
case 'email-dst':
case 'target-email':
case 'whois-registrant-email':
case 'dns-soa-email':
case 'jabber-id':
// we don't use the native function to prevent issues with partial email addresses
if (preg_match("#^.*\@.*\..*$#i", $value)) {
return true;
}
return __('Email address has an invalid format. Please double check the value or select type "other".');
case 'vulnerability':
if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) {
return true;
}
return __('Invalid format. Expected: CVE-xxxx-xxxx...');
case 'weakness':
if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) {
return true;
}
return __('Invalid format. Expected: CWE-x...');
case 'windows-service-name':
case 'windows-service-displayname':
if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) {
return __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.');
}
return true;
case 'mutex':
case 'process-state':
case 'snort':
case 'bro':
case 'zeek':
case 'community-id':
case 'anonymised':
case 'pattern-in-file':
case 'pattern-in-traffic':
case 'pattern-in-memory':
case 'filename-pattern':
case 'pgp-public-key':
case 'pgp-private-key':
case 'yara':
case 'stix2-pattern':
case 'sigma':
case 'gene':
case 'kusto-query':
case 'mime-type':
case 'identity-card-number':
case 'cookie':
case 'attachment':
case 'malware-sample':
case 'comment':
case 'text':
case 'other':
case 'cpe':
case 'email-attachment':
case 'email-body':
case 'email-header':
case 'first-name':
case 'middle-name':
case 'last-name':
case 'full-name':
return true;
case 'link':
// Moved to a native function whilst still enforcing the scheme as a requirement
return (bool)filter_var($value, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
case 'hex':
return ctype_xdigit($value);
case 'target-user':
case 'campaign-name':
case 'campaign-id':
case 'threat-actor':
case 'target-machine':
case 'target-org':
case 'target-location':
case 'target-external':
case 'email-subject':
case 'malware-type':
// TODO: review url/uri validation
case 'url':
case 'uri':
case 'user-agent':
case 'regkey':
case 'regkey|value':
case 'filename':
case 'pdb':
case 'windows-scheduled-task':
case 'whois-registrant-name':
case 'whois-registrant-org':
case 'whois-registrar':
case 'whois-creation-date':
case 'date-of-birth':
case 'place-of-birth':
case 'gender':
case 'passport-number':
case 'passport-country':
case 'passport-expiration':
case 'redress-number':
case 'nationality':
case 'visa-number':
case 'issue-date-of-the-visa':
case 'primary-residence':
case 'country-of-residence':
case 'special-service-request':
case 'frequent-flyer-number':
case 'travel-details':
case 'payment-details':
case 'place-port-of-original-embarkation':
case 'place-port-of-clearance':
case 'place-port-of-onward-foreign-destination':
case 'passenger-name-record-locator-number':
case 'email-dst-display-name':
case 'email-src-display-name':
case 'email-reply-to':
case 'email-x-mailer':
case 'email-mime-boundary':
case 'email-thread-index':
case 'email-message-id':
case 'github-username':
case 'github-repository':
case 'github-organisation':
case 'twitter-id':
case 'dkim':
case 'dkim-signature':
case 'favicon-mmh3':
case 'chrome-extension-id':
case 'mobile-application-id':
case 'named pipe':
if (strpos($value, "\n") !== false) {
return __('Value must not contain new line character.');
}
return true;
case 'ssh-fingerprint':
if (self::isSshFingerprint($value)) {
return true;
}
return __('SSH fingerprint must be in MD5 or SHA256 format.');
case 'datetime':
if (strtotime($value) !== false) {
return true;
}
return __('Datetime has to be in the ISO 8601 format.');
case 'size-in-bytes':
case 'counter':
if (self::isPositiveInteger($value)) {
return true;
}
return __('The value has to be a whole number greater or equal 0.');
/* case 'targeted-threat-index':
if (!is_numeric($value) || $value < 0 || $value > 10) {
return __('The value has to be a number between 0 and 10.');
}
return true;*/
case 'iban':
case 'bic':
case 'btc':
case 'dash':
case 'xmr':
return preg_match('/^[a-zA-Z0-9]+$/', $value) === 1;
case 'vhash':
return preg_match('/^.+$/', $value) === 1;
case 'bin':
case 'cc-number':
case 'bank-account-nr':
case 'aba-rtn':
case 'prtn':
case 'phone-number':
case 'whois-registrant-phone':
case 'float':
return is_numeric($value);
case 'cortex':
json_decode($value);
return json_last_error() === JSON_ERROR_NONE;
case 'boolean':
return $value == 1 || $value == 0;
case 'AS':
if (self::isPositiveInteger($value) && $value <= 4294967295) {
return true;
}
return __('AS number have to be integer between 1 and 4294967295');
}
throw new InvalidArgumentException("Unknown type $type.");
}
/**
* @param string $value
* @return bool
*/
private static function isDomainValid($value)
{
return preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}$#i", $value) === 1;
}
/**
* @param string $value
* @return bool
*/
private static function isPortValid($value)
{
return self::isPositiveInteger($value) && $value >= 1 && $value <= 65535;
}
/**
* @param string $value
* @return bool
*/
private static function isTlshValid($value)
{
if ($value[0] === 't') {
$value = substr($value, 1);
}
return strlen($value) > 35 && ctype_xdigit($value);
}
/**
* @param string $type
* @param string $value
* @return bool
*/
private static function isHashValid($type, $value)
{
return strlen($value) === self::HASH_HEX_LENGTH[$type] && ctype_xdigit($value);
}
/**
* Returns true if input value is positive integer or zero.
* @param int|string $value
* @return bool
*/
private static function isPositiveInteger($value)
{
return (is_int($value) && $value >= 0) || ctype_digit($value);
}
/**
* @param $value
* @return bool
*/
private static function isSsdeep($value)
{
$parts = explode(':', $value);
if (count($parts) !== 3) {
return false;
}
return self::isPositiveInteger($parts[0]);
}
/**
* @param string $value
* @return bool
*/
private static function isSshFingerprint($value)
{
if (substr($value, 0, 7) === 'SHA256:') {
$value = substr($value, 7);
$decoded = base64_decode($value, true);
return $decoded && strlen($decoded) === 32;
} else if (substr($value, 0, 4) === 'MD5:') {
$value = substr($value, 4);
}
$value = str_replace(':', '', $value);
return self::isHashValid('md5', $value);
}
/**
* @param string $value
* @return string
*/
private static function compressIpv6($value)
{
if (strpos($value, ':') && $converted = inet_pton($value)) {
return inet_ntop($converted);
}
return $value;
}
/**
* Temporary solution for utf8 columns until we migrate to utf8mb4.
* via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars
* @param string $input
* @return string
*/
private static function handle4ByteUnicode($input)
{
return preg_replace(
'%(?:
\xF0[\x90-\xBF][\x80-\xBF]{2}
| [\xF1-\xF3][\x80-\xBF]{3}
| \xF4[\x80-\x8F][\x80-\xBF]{2}
)%xs',
'?',
$input
);
}
}

211
app/Lib/Tools/BackgroundJobs/BackgroundJob.php Normal file
View File

@ -0,0 +1,211 @@
<?php
declare(strict_types=1);
class BackgroundJob implements JsonSerializable
{
const
STATUS_WAITING = 1,
STATUS_RUNNING = 2,
STATUS_FAILED = 3,
STATUS_COMPLETED = 4;
/** @var string */
private $id;
/** @var string */
private $command;
/** @var array */
private $args;
/**
* Creation time (UNIX timestamp)
*
* @var integer
*/
private $createdAt;
/**
* Last update time (UNIX timestamp)
*
* @var integer|null
*/
private $updatedAt;
/**@var integer */
private $status;
/** @var integer */
private $progress;
/** @var string|null */
private $output;
/** @var string|null */
private $error;
/** @var array */
private $metadata;
/** @var integer */
private $returnCode;
public function __construct(array $properties)
{
$this->id = $properties['id'];
$this->command = $properties['command'];
$this->args = $properties['args'] ?? [];
$this->createdAt = $properties['createdAt'] ?? time();
$this->updatedAt = $properties['updatedAt'] ?? null;
$this->status = $properties['status'] ?? self::STATUS_WAITING;
$this->error = $properties['error'] ?? null;
$this->progress = $properties['progress'] ?? 0;
$this->metadata = $properties['metadata'] ?? [];
}
/**
* Run the job command
*
* @return self
*/
public function run(): self
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$process = proc_open(
array_merge(
[
ROOT . DS . 'app' . DS . 'Console' . DS . 'cake',
$this->command(),
],
$this->args()
),
$descriptorSpec,
$pipes,
null,
['BACKGROUND_JOB_ID' => $this->id]
);
$stdout = stream_get_contents($pipes[1]);
$this->setOutput($stdout);
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
$this->setError($stderr);
fclose($pipes[2]);
$this->returnCode = proc_close($process);
if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
$this->setProgress(100);
CakeLog::info("[JOB ID: {$this->id()}] - completed.");
} else {
$this->setStatus(BackgroundJob::STATUS_FAILED);
CakeLog::error("[JOB ID: {$this->id()}] - failed with error code {$this->returnCode}. STDERR: {$stderr}. STDOUT: {$stdout}.");
}
return $this;
}
public function jsonSerialize(): array
{
return [
'id' => $this->id,
'command' => $this->command,
'args' => $this->args,
'createdAt' => $this->createdAt,
'updatedAt' => $this->updatedAt,
'status' => $this->status,
'output' => $this->output,
'error' => $this->error,
'metadata' => $this->metadata,
];
}
public function id(): string
{
return $this->id;
}
public function command(): string
{
return $this->command;
}
public function args(): array
{
return $this->args;
}
public function progress(): int
{
return $this->progress;
}
public function createdAt(): int
{
return $this->createdAt;
}
public function updatedAt(): ?int
{
return $this->updatedAt;
}
public function status(): int
{
return $this->status;
}
public function output(): ?string
{
return $this->output;
}
public function error(): ?string
{
return $this->error;
}
public function metadata(): array
{
return $this->metadata;
}
public function returnCode(): int
{
return $this->returnCode;
}
public function setStatus(int $status)
{
$this->status = $status;
}
public function setOutput(?string $output)
{
$this->output = $output;
}
public function setError(?string $error)
{
$this->error = $error;
}
public function setProgress(int $progress)
{
$this->progress = $progress;
}
public function setUpdatedAt(int $updatedAt)
{
$this->updatedAt = $updatedAt;
}
}

107
app/Lib/Tools/BackgroundJobs/Worker.php Normal file
View File

@ -0,0 +1,107 @@
<?php
declare(strict_types=1);
class Worker implements JsonSerializable
{
/** @var integer|null */
private $pid;
/** @var string */
private $queue;
/**
* OS user
*
* @var string|null
*/
private $user;
/**
* creation time (UNIX timestamp)
*
* @var integer
*/
private $createdAt;
/**
* last update time (UNIX timestamp)
*
* @var integer|null
*/
private $updatedAt;
/**
* status id
*
* @var integer
*/
private $status;
const
STATUS_RUNNING = 1,
STATUS_FAILED = 2,
STATUS_UNKNOWN = 3;
public function __construct(array $properties)
{
$this->pid = $properties['pid'];
$this->queue = $properties['queue'];
$this->user = $properties['user'];
$this->createdAt = $properties['createdAt'] ?? time();
$this->updatedAt = $properties['updatedAt'] ?? null;
$this->status = $properties['status'] ?? self::STATUS_UNKNOWN;
}
public function jsonSerialize(): array
{
return [
'pid' => $this->pid,
'queue' => $this->queue,
'user' => $this->user,
'createdAt' => $this->createdAt,
'updatedAt' => $this->updatedAt,
'status' => $this->status,
];
}
public function pid(): ?int
{
return $this->pid;
}
public function queue(): string
{
return $this->queue;
}
public function user(): ?string
{
return $this->user;
}
public function createdAt(): int
{
return $this->createdAt;
}
public function updatedAt(): ?int
{
return $this->updatedAt;
}
public function status(): int
{
return $this->status;
}
public function setStatus(int $status)
{
$this->status = $status;
}
public function setUpdatedAt(int $updatedAt)
{
$this->updatedAt = $updatedAt;
}
}

698
app/Lib/Tools/BackgroundJobsTool.php Normal file
View File

@ -0,0 +1,698 @@
<?php
declare(strict_types=1);
App::uses('Worker', 'Tools/BackgroundJobs');
App::uses('BackgroundJob', 'Tools/BackgroundJobs');
/**
* BackgroundJobs Tool
*
* Utility class to queue jobs, run them and monitor workers.
*
* To run a worker manually (debug only):
* $ ./Console/cake start_worker [queue]
*
* It is recommended to run these commands with [Supervisor](http://supervisord.org).
* `Supervisor` has an extensive feature set to manage scripts as services,
* such as autorestart, parallel execution, logging, monitoring and much more.
* All can be managed via the terminal or a XML-RPC API.
*
* Use the following configuration as a template for the services:
* /etc/supervisor/conf.d/misp-workers.conf:
* [group:misp-workers]
* programs=default,email,cache,prio,update
*
* ; one section per each queue type is required
* [program:default]
* command=/var/www/MISP/app/Console/cake start_worker default
* process_name=%(program_name)s_%(process_num)02d
* numprocs=5 ; adjust the amount of parallel workers to your MISP usage
* autostart=true
* autorestart=true
* redirect_stderr=false
* stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
* stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
* user=www-data
*
*/
class BackgroundJobsTool
{
/** @var Redis */
private $RedisConnection;
/** @var \Supervisor\Supervisor */
private $Supervisor;
const MISP_WORKERS_PROCESS_GROUP = 'misp-workers';
const
STATUS_RUNNING = 0,
STATUS_NOT_ENABLED = 1,
STATUS_REDIS_NOT_OK = 2,
STATUS_SUPERVISOR_NOT_OK = 3,
STATUS_REDIS_AND_SUPERVISOR_NOT_OK = 4;
const
DEFAULT_QUEUE = 'default',
EMAIL_QUEUE = 'email',
CACHE_QUEUE = 'cache',
PRIO_QUEUE = 'prio',
UPDATE_QUEUE = 'update';
const VALID_QUEUES = [
self::DEFAULT_QUEUE,
self::EMAIL_QUEUE,
self::CACHE_QUEUE,
self::PRIO_QUEUE,
self::UPDATE_QUEUE,
];
const
CMD_EVENT = 'event',
CMD_SERVER = 'server',
CMD_ADMIN = 'admin';
const ALLOWED_COMMANDS = [
self::CMD_EVENT,
self::CMD_SERVER,
self::CMD_ADMIN
];
const CMD_TO_SHELL_DICT = [
self::CMD_EVENT => 'EventShell',
self::CMD_SERVER => 'ServerShell',
self::CMD_ADMIN => 'AdminShell'
];
const JOB_STATUS_PREFIX = 'job_status';
/** @var array */
private $settings;
/**
* Initialize
*
* Settings should have the following format:
* [
* 'enabled' => true,
* 'redis_host' => 'localhost',
* 'redis_port' => 6379,
* 'redis_password' => '',
* 'redis_database' => 1,
* 'redis_namespace' => 'background_jobs',
* 'max_job_history_ttl' => 86400
* 'supervisor_host' => 'localhost',
* 'supervisor_port' => '9001',
* 'supervisor_user' => '',
* 'supervisor_password' => '',
* ]
*
* @param array $settings
*/
public function __construct(array $settings)
{
$this->settings = $settings;
if ($this->settings['enabled'] === true) {
$this->RedisConnection = $this->createRedisConnection();
}
}
/**
* Enqueue a Job.
*
* @param string $queue Queue name, e.g. 'default'.
* @param string $command Command of the job.
* @param array $args Arguments passed to the job.
* @param boolean|null $trackStatus Whether to track the status of the job.
* @param int|null $jobId Id of the relational database record representing the job.
* @param array $metadata Related to the job.
* @return string Background Job ID.
* @throws InvalidArgumentException
*/
public function enqueue(
string $queue,
string $command,
array $args = [],
$trackStatus = null,
int $jobId = null,
array $metadata = []
): string {
if (!$this->settings['enabled']) {
return $this->resqueEnqueue($queue, self::CMD_TO_SHELL_DICT[$command], $args, $trackStatus, $jobId);
}
$this->validateQueue($queue);
$this->validateCommand($command);
$backgroundJob = new BackgroundJob(
[
'id' => CakeText::uuid(),
'command' => $command,
'args' => $args,
'metadata' => $metadata
]
);
$this->RedisConnection->rpush(
$queue,
$backgroundJob
);
$this->update($backgroundJob);
if ($jobId) {
$this->updateJobProcessId($jobId, $backgroundJob->id());
}
return $backgroundJob->id();
}
/**
* Enqueue a Job using the CakeResque.
* @deprecated
*
* @param string $queue Name of the queue to enqueue the job to.
* @param string $class Class of the job.
* @param array $args Arguments passed to the job.
* @param boolean $trackStatus Whether to track the status of the job.
* @param int|null $jobId Id of the relational database record representing the job.
* @return string Job Id.
*/
private function resqueEnqueue(
string $queue,
string $class,
$args = [],
$trackStatus = null,
int $jobId = null
): string {
$process_id = CakeResque::enqueue(
$queue,
$class,
$args,
$trackStatus
);
if ($jobId) {
$this->updateJobProcessId($jobId, $process_id);
}
return $process_id;
}
/**
* Dequeue a BackgroundJob.
* If the queue is empty the read is blocked until a job is pushed to this queue or the timeout is reached.
*
* @param string $queue Queue name, e.g. 'default'.
* @param int $timeout Time to block the read if the queue is empty.
* Must be less than your configured `read_write_timeout`
* for the redis connection.
*
* @throws Exception
*/
public function dequeue($queue, int $timeout = 30)
{
$this->validateQueue($queue);
$rawJob = $this->RedisConnection->blpop($queue, $timeout);
if (!empty($rawJob)) {
return new BackgroundJob($rawJob[1]);
}
return null;
}
/**
* Get the job status.
*
* @param string $jobId Background Job Id.
*
*
*/
public function getJob(string $jobId)
{
$rawJob = $this->RedisConnection->get(
self::JOB_STATUS_PREFIX . ':' . $jobId
);
if ($rawJob) {
return new BackgroundJob($rawJob);
}
return null;
}
/**
* Get the queues's names.
*
* @return array Array containing the queues' names.
*/
public function getQueues(): array
{
return self::VALID_QUEUES;
}
/**
* Clear all the queue's jobs.
*
* @param string $queue Queue name, e.g. 'default'.
*
* @return boolean True on success, false on failure.
*/
public function clearQueue($queue): bool
{
$this->validateQueue($queue);
return (bool) $this->RedisConnection->del($queue);
}
/**
* Get all workers' instances.
*
* @return Worker[] List of worker's instances.
*/
public function getWorkers(): array
{
$workers = [];
$procs = $this->getSupervisor()->getAllProcesses();
foreach ($procs as $proc) {
if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) {
if ($proc->offsetGet('pid') > 0) {
$workers[] = new Worker([
'pid' => $proc->offsetGet('pid'),
'queue' => explode("_", $proc->offsetGet('name'))[0],
'user' => $this->processUser((int) $proc->offsetGet('pid')),
'createdAt' => $proc->offsetGet('start'),
'updatedAt' => $proc->offsetGet('now'),
'status' => $this->convertProcessStatus($proc->offsetGet('state'))
]);
}
}
}
return $workers;
}
/**
* Get the number of jobs inside a queue.
*
* @param string $queue Queue name, e.g. 'default'.
*
* @return integer Number of jobs.
*/
public function getQueueSize(string $queue): int
{
$this->validateQueue($queue);
if (!$this->settings['enabled']) {
return CakeResque::getQueueSize($queue);
}
return $this->RedisConnection->llen($queue);
}
/**
* Update job
*
* @param BackgroundJob $job
*
* @return void
*/
public function update(BackgroundJob $job)
{
$job->setUpdatedAt(time());
$this->RedisConnection->setex(
self::JOB_STATUS_PREFIX . ':' . $job->id(),
$this->settings['max_job_history_ttl'],
$job
);
}
/**
* Run job
*
* @param BackgroundJob $job
*
* @return integer Process return code.
*/
public function run(BackgroundJob $job): int
{
$job->setStatus(BackgroundJob::STATUS_RUNNING);
CakeLog::info("[JOB ID: {$job->id()}] - started.");
$this->update($job);
$job = $job->run();
$this->update($job);
return $job->returnCode();
}
/**
* Start worker by name
*
* @param string $name
* @param boolean $waitForRestart
* @return boolean
*/
public function startWorker(string $name, bool $waitForRestart = false): bool
{
$this->validateWorkerName($name);
return $this->getSupervisor()->startProcess(
sprintf(
'%s:%s',
self::MISP_WORKERS_PROCESS_GROUP,
$name
),
$waitForRestart
);
}
/**
* Start worker by queue
*
* @param string $name
* @param boolean $waitForRestart
* @return boolean
*/
public function startWorkerByQueue(string $queue, bool $waitForRestart = false): bool
{
$this->validateQueue($queue);
$procs = $this->getSupervisor()->getAllProcesses();
foreach ($procs as $proc) {
if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) {
$name = explode("_", $proc->offsetGet('name'))[0];
if ($name === $queue && $proc->offsetGet('state') != \Supervisor\Process::RUNNING) {
return $this->getSupervisor()->startProcess(
sprintf(
'%s:%s',
self::MISP_WORKERS_PROCESS_GROUP,
$proc->offsetGet('name')
),
$waitForRestart
);
}
}
}
return false;
}
/**
* Stop worker by name or pid
*
* @param string|int $id
* @param boolean $waitForRestart
* @return boolean
*/
public function stopWorker($id, bool $waitForRestart = false): bool
{
if (is_numeric($id)) {
$process = $this->getProcessByPid((int)$id);
$name = $process->offsetGet('name');
} else {
$name = $id;
}
$this->validateWorkerName($name);
return $this->getSupervisor()->stopProcess(
sprintf(
'%s:%s',
self::MISP_WORKERS_PROCESS_GROUP,
$name
),
$waitForRestart
);
}
/**
* Restarts workers
*
* @param boolean $waitForRestart
* @return void
*/
public function restartWorkers(bool $waitForRestart = false)
{
$this->getSupervisor()->stopProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
* Restarts workers with status != RUNNING
*
* @param boolean $waitForRestart
* @return void
*/
public function restartDeadWorkers(bool $waitForRestart = false)
{
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
* Purge queue
*
* @param string $queue
* @return void
*/
public function purgeQueue(string $queue)
{
$this->validateQueue($queue);
$this->RedisConnection->del($queue);
}
/**
* Return Background Jobs status
*
* @return integer
*/
public function getStatus(): int
{
if (!$this->settings['enabled']) {
return self::STATUS_NOT_ENABLED;
}
try {
$redisStatus = $this->RedisConnection->ping();
} catch (Exception $exception) {
CakeLog::error("SimpleBackgroundJobs Redis error: {$exception->getMessage()}");
$redisStatus = false;
}
try {
$supervisorStatus = $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING;
} catch (Exception $exception) {
CakeLog::error("SimpleBackgroundJobs Supervisor error: {$exception->getMessage()}");
$supervisorStatus = false;
}
if ($redisStatus && $supervisorStatus) {
return self::STATUS_RUNNING;
} elseif (!$redisStatus && !$supervisorStatus) {
return self::STATUS_REDIS_AND_SUPERVISOR_NOT_OK;
} elseif ($redisStatus && !$supervisorStatus) {
return self::STATUS_SUPERVISOR_NOT_OK;
} else {
return self::STATUS_REDIS_NOT_OK;
}
}
/**
* Validate queue
*
* @return boolean
* @throws InvalidArgumentException
*/
private function validateQueue(string $queue): bool
{
if (!in_array($queue, self::VALID_QUEUES, true)) {
throw new InvalidArgumentException(
sprintf(
'Invalid background job queue %s, must be one of: [%s]',
$queue,
implode(', ', self::VALID_QUEUES)
)
);
}
return true;
}
/**
* Validate command
*
* @return boolean
* @throws InvalidArgumentException
*/
private function validateCommand(string $command): bool
{
if (!in_array($command, self::ALLOWED_COMMANDS, true)) {
throw new InvalidArgumentException(
sprintf(
'Invalid command %s, must be one of: [%s]',
$command,
implode(', ', self::ALLOWED_COMMANDS)
)
);
}
return true;
}
/**
* Validate worker name
*
* @param string $name
* @return boolean
* @throws InvalidArgumentException
*/
private function validateWorkerName(string $name): bool
{
list($queue, $id) = explode('_', $name);
$this->validateQueue($queue);
if (!$this->validateQueue($queue) || !is_numeric($id)) {
throw new InvalidArgumentException("Invalid worker name $name, must be one of format {queue_name}_{process_id}, example: default_00");
}
return true;
}
/**
* @return Redis
*/
private function createRedisConnection(): Redis
{
$redis = new Redis();
$redis->connect($this->settings['redis_host'], $this->settings['redis_port']);
$redis->setOption(Redis::OPT_SERIALIZER, Redis::SERIALIZER_JSON);
$redis->setOption(Redis::OPT_PREFIX, $this->settings['redis_namespace'] . ':');
$redisPassword = $this->settings['redis_password'];
if (!empty($redisPassword)) {
$redis->auth($redisPassword);
}
$redis->select($this->settings['redis_database']);
return $redis;
}
/**
* @return \Supervisor\Supervisor
*/
private function getSupervisor()
{
if (!$this->Supervisor) {
$this->Supervisor = $this->createSupervisorConnection();
}
return $this->Supervisor;
}
/**
* @return \Supervisor\Supervisor
*/
private function createSupervisorConnection(): \Supervisor\Supervisor
{
$httpOptions = [];
if (!empty($this->settings['supervisor_user']) && !empty($this->settings['supervisor_password'])) {
$httpOptions = [
'auth' => [
$this->settings['supervisor_user'],
$this->settings['supervisor_password'],
],
];
}
$client = new \fXmlRpc\Client(
sprintf(
'http://%s:%s/RPC2',
$this->settings['supervisor_host'],
$this->settings['supervisor_port']
),
new \fXmlRpc\Transport\HttpAdapterTransport(
new \Http\Message\MessageFactory\GuzzleMessageFactory(),
new \GuzzleHttp\Client($httpOptions)
)
);
return new \Supervisor\Supervisor($client);
}
private function updateJobProcessId(int $jobId, string $processId)
{
$job = ClassRegistry::init('Job');
$job->id = $jobId;
$job->save(['process_id' => $processId]);
}
/**
* Get Supervisor process by PID
*
* @param integer $pid
* @return \Supervisor\Process
*
* @throws NotFoundException
*/
private function getProcessByPid(int $pid): \Supervisor\Process
{
$procs = $this->getSupervisor()->getAllProcesses();
foreach ($procs as $proc) {
if (
$proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP &&
$proc->offsetGet('pid') === $pid
) {
return $proc;
}
}
throw new NotFoundException("Worker with pid=$pid not found.");
}
/**
* Convert process status to worker status
*
* @param integer $stateId
* @return integer
*/
private function convertProcessStatus(int $stateId): int
{
switch ($stateId) {
case \Supervisor\Process::RUNNING:
return Worker::STATUS_RUNNING;
case \Supervisor\Process::UNKNOWN:
return Worker::STATUS_UNKNOWN;
default:
return Worker::STATUS_FAILED;
}
}
/**
* Get effective user name
* @param int $pid
* @return string
*/
private function processUser(int $pid)
{
if (function_exists('posix_getpwuid') && file_exists("/proc/$pid/status")) {
$content = file_get_contents("/proc/$pid/status");
preg_match("/Uid:\t([0-9]+)\t([0-9]+)/", $content, $matches);
return posix_getpwuid((int)$matches[2])['name'];
} else {
return trim(shell_exec(sprintf("ps -o uname='' -p %s", $pid)) ?? '');
}
}
}

66
app/Lib/Tools/BetterSecurity.php Normal file
View File

@ -0,0 +1,66 @@
<?php
class BetterSecurity
{
const METHOD = 'AES-256-GCM';
const TAG_SIZE = 16;
/**
* @param string $plain
* @param string $key
* @return string
* @throws Exception
*/
public static function encrypt($plain, $key)
{
if (strlen($key) < 32) {
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
}
// Generate the encryption key.
$key = hash('sha256', $key, true);
$ivlen = openssl_cipher_iv_length(self::METHOD);
$iv = openssl_random_pseudo_bytes($ivlen);
if ($iv === false) {
throw new Exception('Could not generate random bytes.');
}
$ciphertext = openssl_encrypt($plain, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
if ($ciphertext === false) {
throw new Exception('Could not encrypt.');
}
return $iv . $tag . $ciphertext;
}
/**
* @param string $cipher
* @param string $key
* @return string
* @throws Exception
*/
public static function decrypt($cipher, $key)
{
if (strlen($key) < 32) {
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
}
if (empty($cipher)) {
throw new Exception('The data to decrypt cannot be empty.');
}
// Generate the encryption key.
$key = hash('sha256', $key, true);
$ivSize = openssl_cipher_iv_length(self::METHOD);
// Split out hmac for comparison
$iv = substr($cipher, 0, $ivSize);
$tag = substr($cipher, $ivSize, self::TAG_SIZE);
$cipher = substr($cipher, $ivSize + self::TAG_SIZE);
$decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag);
if ($decrypted === false) {
throw new Exception('Could not decrypt. Maybe invalid encryption key?');
}
return $decrypted;
}
}

6
app/Lib/Tools/CustomPaginationTool.php
View File

@ -99,7 +99,6 @@ class CustomPaginationTool
foreach ($sortArray as $k => $sortedElement) {
$sortArray[$k] = $items[$k];
}
$items = array();
$items = $sortArray;
}
if (!$escapeReindex) {
@ -113,11 +112,12 @@ class CustomPaginationTool
$params = $this->createPaginationRules($items, $options, $model, $sort, $focusKey);
$items = $this->sortArray($items, $params, $escapeReindex);
if (!empty($params['options']['focus'])) {
$focus = $params['options']['focus'];
foreach ($items as $k => $item) {
if ($item[$focusKey] == $params['options']['focus']) {
if ($item[$focusKey] === $focus) {
$params['page'] = 1 + intval(floor($k / $params['limit']));
$params['current'] = 1 + ($params['page'] - 1) * $params['limit'];
continue;
break;
}
}
unset($params['options']['focus']);

68
app/Lib/Tools/EncryptedValue.php Normal file
View File

@ -0,0 +1,68 @@
<?php
App::uses('BetterSecurity', 'Tools');
/**
* Class for ondemand encryption of JSON serialized value
*/
class EncryptedValue implements JsonSerializable
{
const ENCRYPTED_MAGIC = "\x1F\x1D";
/** @var string */
private $value;
/** @var bool */
private $isJson;
public function __construct($value, $isJson = false)
{
$this->value = $value;
$this->isJson = $isJson;
}
/**
* @return mixed
* @throws JsonException
* @throws Exception
*/
public function decrypt()
{
$decrypt = BetterSecurity::decrypt(substr($this->value, 2), Configure::read('Security.encryption_key'));
return $this->isJson ? JsonTool::decode($decrypt) : $decrypt;
}
public function __toString()
{
return $this->decrypt();
}
public function jsonSerialize()
{
return $this->decrypt();
}
/**
* Encrypt provided string if encryption is enabled. If not enabled, input value will be returned.
* @param string $value
* @return string
* @throws Exception
*/
public static function encryptIfEnabled($value)
{
$key = Configure::read('Security.encryption_key');
if ($key) {
return EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $key);
}
return $value;
}
/**
* Check if value is encrypted (starts with encrypted magic)
* @param string $value
* @return bool
*/
public static function isEncrypted($value)
{
return substr($value, 0, 2) === EncryptedValue::ENCRYPTED_MAGIC;
}
}

43
app/Lib/Tools/FileAccessTool.php
View File

@ -1,7 +1,23 @@
<?php
App::uses('JsonTool', 'Tools');
class FileAccessTool
{
/**
* @param string $path
* @param int $permissions
* @throws Exception
*/
public static function createFile($path, $permissions = 0600)
{
if (!file_exists($path)) {
if (!touch($path)) {
throw new Exception("Could not create file `$path`.");
}
}
@chmod($path, $permissions); // hide error if current user is not file owner
}
/**
* Creates temporary file, but you have to delete it after use.
* @param string|null $dir
@ -47,6 +63,33 @@ class FileAccessTool
return $content;
}
/**
* @param string $file
* @return mixed
* @throws Exception
*/
public static function readJsonFromFile($file)
{
$content = self::readFromFile($file);
try {
return JsonTool::decode($content);
} catch (Exception $e) {
throw new Exception("Could not decode JSON from file `$file`", 0, $e);
}
}
/**
* @param string $file
* @return string
* @throws Exception
*/
public static function readAndDelete($file)
{
$content = self::readFromFile($file);
self::deleteFile($file);
return $content;
}
/**
* @param string $file
* @param mixed $content

125
app/Lib/Tools/GitTool.php Normal file
View File

@ -0,0 +1,125 @@
<?php
App::uses('ProcessTool', 'Tools');
class GitTool
{
/**
* @param HttpSocketExtended $HttpSocket
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public static function getLatestTags(HttpSocketExtended $HttpSocket)
{
$url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
$response = $HttpSocket->get($url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
}
/**
* @param HttpSocketExtended $HttpSocket
* @return string
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public static function getLatestCommit(HttpSocketExtended $HttpSocket)
{
$url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
$response = $HttpSocket->get($url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
$data = $response->json();
if (!isset($data[0]['sha'])) {
throw new Exception("Response do not contains requested data.");
}
return $data[0]['sha'];
}
/**
* `git rev-parse HEAD`
* @return string
* @throws Exception
*/
public static function currentCommit()
{
$head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD'));
if (substr($head, 0, 5) === 'ref: ') {
$path = substr($head, 5);
return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path));
} else if (strlen($head) === 40) {
return $head;
} else {
throw new Exception("Invalid head $head");
}
}
/**
* `git symbolic-ref HEAD`
* @return string
* @throws Exception
*/
public static function currentBranch()
{
$head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD'));
if (substr($head, 0, 5) === 'ref: ') {
$path = substr($head, 5);
return str_replace('refs/heads/', '', $path);
} else {
throw new Exception("ref HEAD is not a symbolic ref");
}
}
/**
* @return array
* @throws Exception
*/
public static function submoduleStatus()
{
$lines = ProcessTool::execute(['git', 'submodule', 'status', '--cached'], ROOT);
$output = [];
foreach (explode("\n", $lines) as $submodule) {
if ($submodule === '' || $submodule[0] === '-') {
continue;
}
$parts = explode(' ', $submodule);
$output[] = [
'name' => $parts[2],
'commit' => $parts[1],
];
}
return $output;
}
/**
* @param string $submodule Path to Git repo
* @return string|null
*/
public static function submoduleCurrentCommit($submodule)
{
try {
$commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule);
} catch (ProcessException $e) {
return null;
}
return rtrim($commit);
}
/**
* @param string $commit
* @param string|null $submodule Path to Git repo
* @return int|null
*/
public static function commitTimestamp($commit, $submodule = null)
{
try {
$timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
} catch (ProcessException $e) {
return null;
}
return (int)rtrim($timestamp);
}
}

9
app/Lib/Tools/JSONConverterTool.php
View File

@ -48,7 +48,7 @@ class JSONConverterTool
}
}
if (isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) {
if (empty($event['Event']['SharingGroup'])) {
unset($event['Event']['SharingGroup']);
}
@ -86,11 +86,6 @@ class JSONConverterTool
}
unset($tempSightings);
unset($event['Event']['RelatedAttribute']);
if (isset($event['Event']['RelatedEvent'])) {
foreach ($event['Event']['RelatedEvent'] as $key => $value) {
unset($event['Event']['RelatedEvent'][$key]['Event']['user_id']);
}
}
$result = array('Event' => $event['Event']);
if (isset($event['errors'])) {
$result = array_merge($result, array('errors' => $event['errors']));
@ -143,7 +138,7 @@ class JSONConverterTool
{
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
foreach ($attributes as $key => $attribute) {
if (isset($attribute['SharingGroup']) && empty($attribute['SharingGroup'])) {
if (empty($attribute['SharingGroup'])) {
unset($attributes[$key]['SharingGroup']);
}
unset($attributes[$key]['value1']);

32
app/Lib/Tools/JsonTool.php Normal file
View File

@ -0,0 +1,32 @@
<?php
class JsonTool
{
/**
* @param mixed $value
* @param bool $prettyPrint
* @returns string
* @throws JsonException
*/
public static function encode($value, $prettyPrint = false)
{
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
if (defined('JSON_THROW_ON_ERROR')) {
$flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
}
if ($prettyPrint) {
$flags |= JSON_PRETTY_PRINT;
}
return json_encode($value, $flags);
}
/**
* @param string $value
* @returns mixed
* @throws JsonException
*/
public static function decode($value)
{
$flags = defined('JSON_THROW_ON_ERROR') ? JSON_THROW_ON_ERROR : 0; // Throw exception on error if supported
return json_decode($value, true, 512, $flags);
}
}

120
app/Lib/Tools/ProcessTool.php Normal file
View File

@ -0,0 +1,120 @@
<?php
class ProcessException extends Exception
{
/** @var string */
private $stderr;
/** @var string */
private $stdout;
/**
* @param string|array $command
* @param int $returnCode
* @param string $stderr
* @param string $stdout
*/
public function __construct($command, $returnCode, $stderr, $stdout)
{
$commandForException = is_array($command) ? implode(' ', $command) : $command;
$message = "Command '$commandForException' return error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$this->stderr = $stderr;
$this->stdout = $stdout;
parent::__construct($message, $returnCode);
}
public function stderr()
{
return $this->stderr;
}
public function stdout()
{
return $this->stdout;
}
}
class ProcessTool
{
const LOG_FILE = APP . 'tmp/logs/exec-errors.log';
/**
* @param array $command If command is array, it is not necessary to escape arguments
* @param string|null $cwd
* @param bool $stderrToFile IF true, log stderrr output to LOG_FILE
* @return string Stdout
* @throws ProcessException
* @throws Exception
*/
public static function execute(array $command, $cwd = null, $stderrToFile = false)
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
if ($stderrToFile) {
self::logMessage('Running command ' . implode(' ', $command));
$descriptorSpec[2] = ["file", self::LOG_FILE, 'a'];
}
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
if (PHP_VERSION_ID < 70400) {
$command = array_map('escapeshellarg', $command);
$command = implode(' ', $command);
}
$process = proc_open($command, $descriptorSpec, $pipes, $cwd);
if (!$process) {
$commandForException = self::commandFormat($command);
throw new Exception("Command '$commandForException' could be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
$commandForException = self::commandFormat($command);
throw new Exception("Could not get STDOUT of command '$commandForException'.");
}
fclose($pipes[1]);
if ($stderrToFile) {
$stderr = null;
} else {
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
}
$returnCode = proc_close($process);
if ($stderrToFile) {
self::logMessage("Process finished with return code $returnCode");
}
if ($returnCode !== 0) {
throw new ProcessException($command, $returnCode, $stderr, $stdout);
}
return $stdout;
}
/**
* @return string
*/
public static function pythonBin()
{
return Configure::read('MISP.python_bin') ?: 'python3';
}
private static function logMessage($message)
{
$logMessage = '[' . date("Y-m-d H:i:s") . ' ' . getmypid() . "] $message\n";
file_put_contents(self::LOG_FILE, $logMessage, FILE_APPEND | LOCK_EX);
}
/**
* @param array|string $command
* @return string
*/
private static function commandFormat($command)
{
return is_array($command) ? implode(' ', $command) : $command;
}
}

90
app/Lib/Tools/PubSubTool.php
View File

@ -1,4 +1,8 @@
<?php
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
App::uses('ProcessTool', 'Tools');
class PubSubTool
{
const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
@ -9,18 +13,12 @@ class PubSubTool
*/
private $redis;
/**
* @var array
*/
private $settings;
public function initTool()
{
if (!$this->redis) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
$this->redis = $this->createRedisConnection($settings);
$this->settings = $settings;
}
}
@ -35,17 +33,19 @@ class PubSubTool
*/
public function checkIfRunning($pidFilePath = null)
{
$pidFile = new File($pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid');
if (!$pidFile->exists()) {
$pidFile = $pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid';
clearstatcache(false, $pidFile);
if (!file_exists($pidFile)) {
return false;
}
$pid = $pidFile->read();
$pid = file_get_contents($pidFile);
if ($pid === false || $pid === '') {
return false;
}
if (!is_numeric($pid)) {
throw new Exception('Internal error (invalid PID file for the MISP zmq script)');
}
clearstatcache(false, "/proc/$pid");
$result = file_exists("/proc/$pid");
if ($result === false) {
return false;
@ -57,8 +57,8 @@ class PubSubTool
{
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'status');
$response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
$redis->rPush( 'command', 'status');
$response = $redis->blPop('status', 5);
if ($response === null) {
throw new Exception("No response from status command returned after 5 seconds.");
}
@ -67,9 +67,9 @@ class PubSubTool
public function checkIfPythonLibInstalled()
{
$my_server = ClassRegistry::init('Server');
$result = trim(shell_exec($my_server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py'));
if ($result === "OK") {
$script = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py';
$result = ProcessTool::execute([ProcessTool::pythonBin(), $script]);
if (trim($result) === "OK") {
return true;
}
return false;
@ -79,8 +79,8 @@ class PubSubTool
{
App::uses('JSONConverterTool', 'Tools');
$jsonTool = new JSONConverterTool();
$json = $jsonTool->convert($event);
return $this->pushToRedis(':data:misp_json', $json);
$json = $jsonTool->convert($event, false, true);
return $this->pushToRedis('data:misp_json', $json);
}
public function event_save(array $event, $action)
@ -88,7 +88,7 @@ class PubSubTool
if (!empty($action)) {
$event['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_event', $event);
return $this->pushToRedis('data:misp_json_event', $event);
}
public function object_save(array $object, $action)
@ -96,7 +96,7 @@ class PubSubTool
if (!empty($action)) {
$object['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_object', $object);
return $this->pushToRedis('data:misp_json_object', $object);
}
public function object_reference_save(array $object_reference, $action)
@ -104,12 +104,12 @@ class PubSubTool
if (!empty($action)) {
$object_reference['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_object_reference', $object_reference);
return $this->pushToRedis('data:misp_json_object_reference', $object_reference);
}
public function publishConversation(array $message)
{
return $this->pushToRedis(':data:misp_json_conversation', $message);
return $this->pushToRedis('data:misp_json_conversation', $message);
}
public function attribute_save(array $attribute, $action = false)
@ -117,7 +117,7 @@ class PubSubTool
if (!empty($action)) {
$attribute['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_attribute', $attribute);
return $this->pushToRedis('data:misp_json_attribute', $attribute);
}
public function tag_save(array $tag, $action = false)
@ -125,7 +125,7 @@ class PubSubTool
if (!empty($action)) {
$tag['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_tag', $tag);
return $this->pushToRedis('data:misp_json_tag', $tag);
}
public function sighting_save(array $sighting, $action = false)
@ -133,7 +133,7 @@ class PubSubTool
if (!empty($action)) {
$sighting['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_sighting', $sighting);
return $this->pushToRedis('data:misp_json_sighting', $sighting);
}
public function warninglist_save(array $warninglist, $action = false)
@ -141,7 +141,7 @@ class PubSubTool
if (!empty($action)) {
$warninglist['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_warninglist', $warninglist);
return $this->pushToRedis('data:misp_json_warninglist', $warninglist);
}
/**
@ -149,13 +149,14 @@ class PubSubTool
* @param string $type
* @param string|false $action
* @return bool
* @throws JsonException
*/
public function modified($data, $type, $action = false)
{
if (!empty($action)) {
$data['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_' . $type, $data);
return $this->pushToRedis('data:misp_json_' . $type, $data);
}
public function publish($data, $type, $action = false)
@ -163,7 +164,7 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_' . $type, $data);
return $this->pushToRedis('data:misp_json_' . $type, $data);
}
public function killService()
@ -171,7 +172,7 @@ class PubSubTool
if ($this->checkIfRunning()) {
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
$redis->rPush('command', 'kill');
sleep(1);
if ($this->checkIfRunning()) {
// Still running.
@ -194,7 +195,7 @@ class PubSubTool
if ($this->checkIfRunning()) {
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'reload');
$redis->rPush( 'command', 'reload');
} else {
return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
}
@ -226,13 +227,12 @@ class PubSubTool
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
$redis->rPush('command', 'kill');
sleep(1);
}
$this->saveSettingToFile($settings);
$server = ClassRegistry::init('Server');
shell_exec($server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
shell_exec(ProcessTool::pythonBin() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
}
}
@ -240,14 +240,12 @@ class PubSubTool
* @param string $ns
* @param string|array $data
* @return bool
* @throws JsonException
*/
private function pushToRedis($ns, $data)
{
if (is_array($data)) {
$data = json_encode($data, JSON_UNESCAPED_UNICODE);
}
$this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
$data = JsonTool::encode($data);
$this->redis->rPush($ns, $data);
return true;
}
@ -264,6 +262,7 @@ class PubSubTool
$redis->auth($redisPassword);
}
$redis->select($settings['redis_database']);
$redis->setOption(Redis::OPT_PREFIX, $settings['redis_namespace'] . ':');
return $redis;
}
@ -274,27 +273,21 @@ class PubSubTool
private function saveSettingToFile(array $settings)
{
$settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
$settingsFile = new File($settingFilePath, true, 0644);
if (!$settingsFile->exists()) {
throw new Exception("Could not create zmq config file '$settingFilePath'.");
}
// Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
// config file is created under one user and then changed under other user, file must be readable and writable
// also by group.
@chmod($settingsFile->pwd(), 0660); // hide error if current user is not file owner
if (!$settingsFile->write(json_encode($settings))) {
throw new Exception("Could not write zmq config file '$settingFilePath'.");
}
$settingsFile->close();
FileAccessTool::createFile($settingFilePath, 0660);
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
}
private function getSetSettings()
{
$settings = array(
'redis_host' => 'localhost',
'redis_port' => '6379',
'redis_port' => 6379,
'redis_password' => '',
'redis_database' => '1',
'redis_database' => 1,
'redis_namespace' => 'mispq',
'host' => '127.0.0.1',
'port' => '50000',
@ -302,8 +295,9 @@ class PubSubTool
'password' => null,
);
$pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
$temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null;
if ($temp) {
$settings[$key] = $temp;
}

35
app/Lib/Tools/SecurityAudit.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('SyncTool', 'Tools');
App::uses('ProcessTool', 'Tools');
class SecurityAudit
{
@ -384,7 +385,7 @@ class SecurityAudit
// uptime
try {
$since = $this->execute(['uptime', '-s']);
$since = ProcessTool::execute(['uptime', '-s']);
$since = new DateTime($since);
$diff = (new DateTime())->diff($since);
$diffDays = $diff->format('a');
@ -399,7 +400,7 @@ class SecurityAudit
// Python version
try {
$pythonVersion = $this->execute([$server->getPythonVersion(), '-V']);
$pythonVersion = ProcessTool::execute([ProcessTool::pythonBin(), '-V']);
$parts = explode(' ', $pythonVersion);
if ($parts[0] !== 'Python') {
throw new Exception("Invalid python version response: $pythonVersion");
@ -499,34 +500,4 @@ class SecurityAudit
}
throw new RuntimeException("CakePHP version not found in file '$filePath'.");
}
private function execute(array $command)
{
$descriptorspec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$command = implode(' ', $command);
$process = proc_open($command, $descriptorspec, $pipes);
if (!$process) {
throw new Exception("Command '$command' could not be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
throw new Exception("Could not get STDOUT of command.");
}
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if ($returnCode !== 0) {
throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
}
return $stdout;
}
}

3
app/Lib/Tools/SendEmail.php
View File

@ -974,6 +974,7 @@ class SendEmail
$parts[] = 'prefer-encrypt=mutual';
}
$parts[] = 'keydata=' . base64_encode($keyData);
return implode('; ', $parts);
// Use the PHP wordwrap function to wrap the Autocrypt header to 74 (+ CRLF) to meet RFC 5322 - 2.1.1 line length limits
return wordwrap(implode('; ', $parts), 74, "\r\n\t", true);
}
}

44
app/Lib/Tools/ServerSyncTool.php
View File

@ -50,7 +50,9 @@ class ServerSyncTool
public function eventExists(array $event)
{
$url = $this->server['Server']['url'] . '/events/view/' . $event['Event']['uuid'];
$start = microtime(true);
$exists = $this->socket->head($url, [], $this->request);
$this->log($start, 'HEAD', $url, $exists);
if ($exists->code == '404') {
return false;
}
@ -83,6 +85,17 @@ class ServerSyncTool
return $this->get($url);
}
/**
* @param array $rules
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function attributeSearch(array $rules)
{
return $this->post('/attributes/restSearch.json', $rules);
}
/**
* @param array $params
* @return HttpSocketResponseExtended
@ -170,6 +183,16 @@ class ServerSyncTool
return $this->get('/users/view/me.json');
}
/**
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function resetAuthKey()
{
return $this->post('/users/resetauthkey/me', []);
}
/**
* @param string $testString
* @return HttpSocketResponseExtended
@ -235,7 +258,9 @@ class ServerSyncTool
private function get($url)
{
$url = $this->server['Server']['url'] . $url;
$start = microtime(true);
$response = $this->socket->get($url, [], $this->request);
$this->log($start, 'GET', $url, $response);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
@ -261,7 +286,7 @@ class ServerSyncTool
$logMessage,
$data
);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND | LOCK_EX);
}
$request = $this->request;
@ -275,7 +300,9 @@ class ServerSyncTool
}
}
$url = $this->server['Server']['url'] . $url;
$start = microtime(true);
$response = $this->socket->post($url, $data, $request);
$this->log($start, 'POST', $url, $response);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
@ -300,4 +327,19 @@ class ServerSyncTool
}
return $url;
}
/**
* @param float $start
* @param string $method HTTP method
* @param string $url
* @param HttpSocketResponse $response
*/
private function log($start, $method, $url, HttpSocketResponse $response)
{
$duration = round(microtime(true) - $start, 3);
$responseSize = strlen($response->body);
$ce = $response->getHeader('Content-Encoding');
$logEntry = '[' . date("Y-m-d H:i:s") . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n";
file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX);
}
}

27
app/Model/AdminSetting.php
View File

@ -18,16 +18,27 @@ class AdminSetting extends AppModel
public function changeSetting($setting, $value = false)
{
$setting_object = $this->find('first', array(
'conditions' => array('setting' => $setting)
$existing = $this->find('first', array(
'conditions' => array('setting' => $setting),
'fields' => ['id'],
));
$this->deleteAll(array('setting' => $setting));
$this->create();
$setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value);
if ($this->save($setting_object)) {
return true;
if ($existing) {
if ($this->save([
'id' => $existing['AdminSetting']['id'],
'value' => $value,
])) {
return true;
} else {
return $this->validationErrors;
}
} else {
return $this->validationErrors;
$this->create();
$existing['AdminSetting'] = array('setting' => $setting, 'value' => $value);
if ($this->save($existing)) {
return true;
} else {
return $this->validationErrors;
}
}
}

545
app/Model/AppModel.php
View File

@ -23,23 +23,26 @@
App::uses('Model', 'Model');
App::uses('LogableBehavior', 'Assets.models/behaviors');
App::uses('RandomTool', 'Tools');
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
class AppModel extends Model
{
public $name;
/** @var PubSubTool */
private static $loadedPubSubTool;
/** @var KafkaPubTool */
private $loadedKafkaPubTool;
/** @var BackgroundJobsTool */
private static $loadedBackgroundJobsTool;
/** @var null|Redis */
private static $__redisConnection;
private $__profiler = array();
public $elasticSearchClient = false;
public $elasticSearchClient;
/** @var AttachmentTool|null */
private $attachmentTool;
@ -47,8 +50,6 @@ class AppModel extends Model
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->name = get_class($this);
$this->findMethods['column'] = true;
}
@ -83,7 +84,8 @@ class AppModel extends Model
51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false,
57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false,
63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false,
69 => false, 70 => false, 71 => true, 72 => true,
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
75 => false, 76 => true, 77 => false, 78 => false, 79 => false,
);
public $advanced_updates_description = array(
@ -246,13 +248,13 @@ class AppModel extends Model
$result = $this->Feed->addDefaultFeeds($feeds);
$this->Log->create();
$entry = array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Added new default feeds.'
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Added new default feeds.'
);
if ($result) {
$entry['change'] = 'Feeds added: ' . $feedNames;
@ -1533,7 +1535,7 @@ class AppModel extends Model
`value` text NOT NULL,
`from_json` tinyint(1) default 0,
PRIMARY KEY (`id`),
UNIQUE INDEX `value` (`value`(255))
UNIQUE INDEX `value` (`value`(191))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
break;
case 66:
@ -1578,6 +1580,40 @@ class AppModel extends Model
case 72:
$sqlArray[] = "ALTER TABLE `auth_keys` ADD `read_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `expiration`;";
break;
case 73:
$this->__dropIndex('user_settings', 'timestamp'); // index is not used
$sqlArray[] = "ALTER TABLE `user_settings` ADD UNIQUE INDEX `unique_setting` (`user_id`, `setting`)";
break;
case 74:
$sqlArray[] = "ALTER TABLE `users` MODIFY COLUMN `change_pw` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 75:
$this->__addIndex('object_references', 'event_id');
$this->__dropIndex('object_references', 'timestamp');
$this->__dropIndex('object_references', 'source_uuid');
$this->__dropIndex('object_references', 'relationship_type');
$this->__dropIndex('object_references', 'referenced_uuid');
break;
case 76:
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `system_settings` (
`setting` varchar(255) NOT NULL,
`value` blob NOT NULL,
PRIMARY KEY (`setting`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
$sqlArray[] = "ALTER TABLE `servers` MODIFY COLUMN `authkey` VARBINARY(255) NOT NULL;";
$sqlArray[] = "ALTER TABLE `cerebrates` MODIFY COLUMN `authkey` VARBINARY(255) NOT NULL;";
break;
case 77:
$sqlArray[] = "ALTER TABLE `tags` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `is_custom_galaxy`;";
$sqlArray[] = "ALTER TABLE `galaxies` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `enabled`;";
break;
case 78:
$sqlArray[] = "ALTER TABLE `jobs` MODIFY COLUMN `process_id` varchar(36) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL;";
break;
case 79:
$sqlArray[] = "ALTER TABLE `users` ADD `sub` varchar(255) NULL DEFAULT NULL;";
$sqlArray[] = "ALTER TABLE `users` ADD UNIQUE INDEX `sub` (`sub`);";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -1767,14 +1803,14 @@ class AppModel extends Model
if ($flagStop && $errorCount > 0) {
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => __('Issues executing the SQL query for %s', $command),
'change' => __('Database updates stopped as some errors occurred and the stop flag is enabled.')
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => __('Issues executing the SQL query for %s', $command),
'change' => __('Database updates stopped as some errors occurred and the stop flag is enabled.')
));
return false;
}
@ -1840,14 +1876,14 @@ class AppModel extends Model
}
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
'change' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
'change' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
));
}
}
@ -1879,14 +1915,14 @@ class AppModel extends Model
}
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
));
$additionResult = array('success' => $result || $duplicate);
if (!$result) {
@ -1914,11 +1950,7 @@ class AppModel extends Model
public function getPythonVersion()
{
if (!empty(Configure::read('MISP.python_bin'))) {
return Configure::read('MISP.python_bin');
} else {
return 'python3';
}
return Configure::read('MISP.python_bin') ?: 'python3';
}
public function validateAuthkey($value)
@ -1935,10 +1967,9 @@ class AppModel extends Model
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value)
{
$field = array_keys($value);
$field = $field[0];
$value[$field] = trim($value[$field]);
if (!empty($value[$field])) {
$field = array_keys($value)[0];
$value = trim($value[$field]);
if (!empty($value)) {
return true;
}
return ucfirst($field) . ' cannot be empty.';
@ -1946,32 +1977,17 @@ class AppModel extends Model
public function valueIsJson($value)
{
$field = array_keys($value);
$field = $field[0];
$json_decoded = json_decode($value[$field]);
$value = array_values($value)[0];
$json_decoded = json_decode($value);
if ($json_decoded === null) {
return __('Invalid JSON.');
}
return true;
}
public function valueIsJsonOrNull($value)
{
$field = array_keys($value);
$field = $field[0];
if (!is_null($value[$field])) {
$json_decoded = json_decode($value[$field]);
if ($json_decoded === null) {
return __('Invalid JSON.');
}
}
return true;
}
public function valueIsID($value)
{
$field = array_keys($value);
$field = $field[0];
$field = array_keys($value)[0];
if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID';
}
@ -1980,17 +1996,17 @@ class AppModel extends Model
public function stringNotEmpty($value)
{
$field = array_keys($value);
$field = $field[0];
$value[$field] = trim($value[$field]);
if (!isset($value[$field]) || ($value[$field] == false && $value[$field] !== "0")) {
$field = array_keys($value)[0];
$value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.';
}
return true;
}
// Try to create a table with a BIGINT(20)
public function seenOnAttributeAndObjectPreUpdate() {
public function seenOnAttributeAndObjectPreUpdate()
{
$sqlArray[] = "CREATE TABLE IF NOT EXISTS testtable (
`testfield` BIGINT(6) NULL DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
@ -2008,16 +2024,11 @@ class AppModel extends Model
}
}
public function failingPreUpdate() {
throw new Exception('Yolo fail');
}
public function runUpdates($verbose = false, $useWorker = true, $processId = false)
{
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$this->Job = ClassRegistry::init('Job');
$this->Log = ClassRegistry::init('Log');
$this->Server = ClassRegistry::init('Server');
$db = ConnectionManager::getDataSource('default');
$tables = $db->listSources();
$requiresLogout = false;
@ -2027,7 +2038,10 @@ class AppModel extends Model
$requiresLogout = true;
} else {
$this->__runCleanDB();
$db_version = $this->AdminSetting->find('all', array('conditions' => array('setting' => 'db_version')));
$db_version = $this->AdminSetting->find('all', [
'conditions' => array('setting' => 'db_version'),
'fields' => ['id', 'value'],
]);
if (count($db_version) > 1) {
// we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
foreach ($db_version as $k => $v) {
@ -2046,6 +2060,8 @@ class AppModel extends Model
$job = null;
}
if (!empty($updates)) {
$this->Log = ClassRegistry::init('Log');
$this->Server = ClassRegistry::init('Server');
// Exit if updates are locked.
// This is not as reliable as a real lock implementation
// However, as all updates are re-playable, there is no harm if they
@ -2054,14 +2070,14 @@ class AppModel extends Model
if ($this->isUpdateLocked()) { // prevent creation of useless workers
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_db_worker',
'user_id' => 0,
'title' => __('Issues executing run_updates'),
'change' => __('Database updates are locked. Worker not spawned')
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_db_worker',
'user_id' => 0,
'title' => __('Issues executing run_updates'),
'change' => __('Database updates are locked. Worker not spawned')
));
if (!empty($job)) { // if multiple prio worker is enabled, want to mark them as done
$job['Job']['progress'] = 100;
@ -2080,26 +2096,28 @@ class AppModel extends Model
} else { // update worker not running, doing the update inline
return $this->runUpdates($verbose, false);
}
$this->Job->create();
$data = array(
'worker' => $workerType,
'job_type' => 'run_updates',
'job_input' => 'command: ' . implode(',', $updates),
'status' => 0,
'retries' => 0,
'org_id' => 0,
'org' => '',
'message' => 'Updating.',
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_PRIO,
'run_updates',
'command: ' . implode(',', $updates),
'Updating.'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'runUpdates',
$jobId
],
true,
$jobId
);
$this->Job->save($data);
$jobId = $this->Job->id;
$processId = CakeResque::enqueue(
'prio',
'AdminShell',
array('runUpdates', $jobId),
true
);
$this->Job->saveField('process_id', $processId);
return true;
}
@ -2109,14 +2127,14 @@ class AppModel extends Model
if ($this->isUpdateLocked()) {
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_db_worker',
'user_id' => 0,
'title' => __('Issues executing run_updates'),
'change' => __('Updates are locked. Stopping worker gracefully')
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_db_worker',
'user_id' => 0,
'title' => __('Issues executing run_updates'),
'change' => __('Updates are locked. Stopping worker gracefully')
));
if (!empty($job)) {
$job['Job']['progress'] = 100;
@ -2367,19 +2385,17 @@ class AppModel extends Model
private function __runCleanDB()
{
$cleanDB = $this->AdminSetting->find('first', array('conditions' => array('setting' => 'clean_db')));
if (empty($cleanDB) || $cleanDB['AdminSetting']['value'] == 1) {
$cleanDB = $this->AdminSetting->getSetting('clean_db');
if ($cleanDB === false || $cleanDB == 1) {
$this->cleanCacheFiles();
if (empty($cleanDB)) {
$this->AdminSetting->create();
$cleanDB = array('AdminSetting' => array('setting' => 'clean_db', 'value' => 0));
} else {
$cleanDB['AdminSetting']['value'] = 0;
}
$this->AdminSetting->save($cleanDB);
$this->AdminSetting->changeSetting('clean_db', 0);
}
}
/**
* @param string $db_version
* @return array
*/
protected function findUpgrades($db_version)
{
$updates = array();
@ -2419,84 +2435,31 @@ class AppModel extends Model
private function __generateCorrelations()
{
if (Configure::read('MISP.background_jobs')) {
$Job = ClassRegistry::init('Job');
$Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'generate correlation',
'job_input' => 'All attributes',
'status' => 0,
'retries' => 0,
'org' => 'ADMIN',
'message' => 'Job created.',
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate correlation',
'All attributes',
'Job created.'
);
$Job->save($data);
$jobId = $Job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('jobGenerateCorrelation', $jobId),
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateCorrelation',
$jobId
],
true,
$jobId
);
$Job->saveField('process_id', $process_id);
}
return true;
}
public function populateNotifications($user, $mode = 'full')
{
$notifications = array();
list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user, $mode);
$notifications['total'] = $notifications['proposalCount'];
if (Configure::read('MISP.delegation')) {
$notifications['delegationCount'] = $this->_getDelegationCount($user);
$notifications['total'] += $notifications['delegationCount'];
}
return $notifications;
}
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount($user, $mode = 'full')
{
$this->ShadowAttribute = ClassRegistry::init('ShadowAttribute');
$results[0] = $this->ShadowAttribute->find(
'count',
array(
'recursive' => -1,
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
)
)
);
if ($mode === 'full') {
$results[1] = $this->ShadowAttribute->find(
'count',
array(
'recursive' => -1,
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
),
'fields' => 'distinct event_id'
)
);
} else {
$results[1] = $results[0];
}
return $results;
}
private function _getDelegationCount($user)
{
$this->EventDelegation = ClassRegistry::init('EventDelegation');
$delegations = $this->EventDelegation->find('count', array(
'recursive' => -1,
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
return $delegations;
}
public function checkFilename($filename)
{
return preg_match('@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i', $filename);
@ -2557,26 +2520,20 @@ class AppModel extends Model
public function getKafkaPubTool()
{
if (!$this->loadedKafkaPubTool) {
$this->loadKafkaPubTool();
App::uses('KafkaPubTool', 'Tools');
$kafkaPubTool = new KafkaPubTool();
$rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config');
$kafkaConf = array();
if (!empty($rdkafkaIni)) {
$kafkaConf = parse_ini_file($rdkafkaIni);
}
$brokers = Configure::read('Plugin.Kafka_brokers');
$kafkaPubTool->initTool($brokers, $kafkaConf);
$this->loadedKafkaPubTool = $kafkaPubTool;
}
return $this->loadedKafkaPubTool;
}
public function loadKafkaPubTool()
{
App::uses('KafkaPubTool', 'Tools');
$kafkaPubTool = new KafkaPubTool();
$rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config');
$kafkaConf = array();
if (!empty($rdkafkaIni)) {
$kafkaConf = parse_ini_file($rdkafkaIni);
}
$brokers = Configure::read('Plugin.Kafka_brokers');
$kafkaPubTool->initTool($brokers, $kafkaConf);
$this->loadedKafkaPubTool = $kafkaPubTool;
return true;
}
public function publishKafkaNotification($topicName, $data, $action = false)
{
$kafkaTopic = $this->kafkaTopic($topicName);
@ -2599,26 +2556,40 @@ class AppModel extends Model
return self::$loadedPubSubTool;
}
public function getElasticSearchTool()
protected function getElasticSearchTool()
{
if (!$this->elasticSearchClient) {
$this->loadElasticSearchTool();
App::uses('ElasticSearchClient', 'Tools');
$client = new ElasticSearchClient();
$client->initTool();
$this->elasticSearchClient = $client;
}
return $this->elasticSearchClient;
}
public function loadElasticSearchTool()
/**
* @return BackgroundJobsTool
*/
public function getBackgroundJobsTool(): BackgroundJobsTool
{
App::uses('ElasticSearchClient', 'Tools');
$client = new ElasticSearchClient();
$client->initTool();
$this->elasticSearchClient = $client;
if (!self::$loadedBackgroundJobsTool) {
App::uses('BackgroundJobsTool', 'Tools');
// TODO: remove after CakeResque is deprecated
$settings = ['enabled' => false];
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$settings = Configure::read('SimpleBackgroundJobs');
}
$backgroundJobsTool = new BackgroundJobsTool($settings);
self::$loadedBackgroundJobsTool = $backgroundJobsTool;
}
return self::$loadedBackgroundJobsTool;
}
// generate a generic subquery - options needs to include conditions
public function subQueryGenerator($model, $options, $lookupKey, $negation = false)
protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false)
{
$db = $model->getDataSource();
$defaults = array(
'fields' => array('*'),
'table' => $model->table,
@ -2631,17 +2602,15 @@ class AppModel extends Model
'recursive' => -1
);
$params = array();
foreach (array_keys($defaults) as $key) {
foreach ($defaults as $key => $defaultValue) {
if (isset($options[$key])) {
$params[$key] = $options[$key];
} else {
$params[$key] = $defaults[$key];
$params[$key] = $defaultValue;
}
}
$subQuery = $db->buildStatement(
$params,
$model
);
$db = $model->getDataSource();
$subQuery = $db->buildStatement($params, $model);
if ($negation) {
$subQuery = $lookupKey . ' NOT IN (' . $subQuery . ') ';
} else {
@ -2712,15 +2681,6 @@ class AppModel extends Model
}
}
public function getRowCount($table = false)
{
if (empty($table)) {
$table = $this->table;
}
$table_data = $this->query("show table status like '" . $table . "'");
return $table_data[0]['TABLES']['Rows'];
}
public function benchmarkCustomAdd($valueToAdd = 0, $name = 'default', $customName = 'custom')
{
if (empty($this->__profiler[$name]['custom'][$customName])) {
@ -2762,15 +2722,21 @@ class AppModel extends Model
{
$version = implode('.', $this->checkMISPVersion());
$commit = $this->checkMIPSCommit();
$request = array(
$authkey = $server[$model]['authkey'];
App::uses('EncryptedValue', 'Tools');
if (EncryptedValue::isEncrypted($authkey)) {
$authkey = (string)new EncryptedValue($authkey);
}
return array(
'header' => array(
'Authorization' => $server[$model]['authkey'],
'Authorization' => $authkey,
'Accept' => 'application/json',
'Content-Type' => 'application/json',
'User-Agent' => 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit),
)
);
return $request;
}
/**
@ -2783,9 +2749,8 @@ class AppModel extends Model
{
static $versionArray;
if ($versionArray === null) {
$file = new File(ROOT . DS . 'VERSION.json');
$versionArray = $this->jsonDecode($file->read());
$file->close();
$content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json');
$versionArray = JsonTool::decode($content);
}
return $versionArray;
}
@ -2799,10 +2764,11 @@ class AppModel extends Model
{
static $commit;
if ($commit === null) {
$commit = shell_exec('git log --pretty="%H" -n1 HEAD');
if ($commit) {
$commit = trim($commit);
} else {
App::uses('GitTool', 'Tools');
try {
$commit = GitTool::currentCommit();
} catch (Exception $e) {
$this->logException('Could not get current git commit', $e, LOG_NOTICE);
$commit = false;
}
}
@ -2929,11 +2895,6 @@ class AppModel extends Model
return $val / (1024 * 1024);
}
public function getDefaultAttachments_dir()
{
return APP . 'files';
}
private function __bumpReferences()
{
$this->Event = ClassRegistry::init('Event');
@ -2991,14 +2952,14 @@ class AppModel extends Model
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$entry = array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Bumped the timestamps of locked events containing object references.',
'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids))
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Bumped the timestamps of locked events containing object references.',
'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids))
);
$this->Log->save($entry);
}
@ -3017,10 +2978,10 @@ class AppModel extends Model
}
$multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60, 's' => 1);
$lastChar = strtolower(substr($delta, -1));
if (!is_numeric($lastChar) && array_key_exists($lastChar, $multiplierArray)) {
if (!is_numeric($lastChar) && isset($multiplierArray[$lastChar])) {
$multiplier = $multiplierArray[$lastChar];
$delta = substr($delta, 0, -1);
} else if(strtotime($delta) !== false) {
} else if (strtotime($delta) !== false) {
return strtotime($delta);
} else {
// invalid filter, make sure we don't return anything
@ -3156,10 +3117,7 @@ class AppModel extends Model
$message .= "\n";
do {
$message .= sprintf("[%s] %s",
get_class($exception),
$exception->getMessage()
);
$message .= sprintf("[%s] %s", get_class($exception), $exception->getMessage());
$message .= "\nStack Trace:\n" . $exception->getTraceAsString();
$exception = $exception->getPrevious();
} while ($exception !== null);
@ -3167,23 +3125,6 @@ class AppModel extends Model
return $this->log($message, $type);
}
/**
* Generates random file name in tmp dir.
* @return string
*/
protected function tempFileName()
{
return $this->tempDir() . DS . $this->generateRandomFileName();
}
/**
* @return string
*/
protected function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
/**
* Decodes JSON string and throws exception if string is not valid JSON or if is not array.
*
@ -3210,23 +3151,6 @@ class AppModel extends Model
return $decoded;
}
/*
* Temporary solution for utf8 columns until we migrate to utf8mb4
* via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars
*/
public function handle4ByteUnicode($input)
{
return preg_replace(
'%(?:
\xF0[\x90-\xBF][\x80-\xBF]{2}
| [\xF1-\xF3][\x80-\xBF]{3}
| \xF4[\x80-\x8F][\x80-\xBF]{2}
)%xs',
'?',
$input
);
}
/**
* Faster version of default `hasAny` method
* @param array|null $conditions
@ -3239,9 +3163,22 @@ class AppModel extends Model
'conditions' => $conditions,
'recursive' => -1,
'callbacks' => false,
'order' => [], // disable order
));
}
/**
* @param int $value Timestamp in microseconds
* @return string
*/
protected function microTimestampToIso($value)
{
$sec = (int)($value / 1000000);
$micro = $value % 1000000;
$micro = str_pad($micro, 6, "0", STR_PAD_LEFT);
return DateTime::createFromFormat('U.u', "$sec.$micro")->format('Y-m-d\TH:i:s.uP');
}
/**
* @return AttachmentTool
*/
@ -3295,4 +3232,20 @@ class AppModel extends Model
}
return null;
}
/**
* @param string $name
* @return bool
*/
protected function pubToZmq($name)
{
static $zmqEnabled;
if ($zmqEnabled === null) {
$zmqEnabled = (bool)Configure::read('Plugin.ZeroMQ_enable');
}
if ($zmqEnabled) {
return Configure::read("Plugin.ZeroMQ_{$name}_notifications_enable");
}
return false;
}
}

24
app/Model/AttachmentScan.php
View File

@ -295,14 +295,26 @@ class AttachmentScan extends AppModel
if ($canScan) {
$job = ClassRegistry::init('Job');
$jobId = $job->createJob('SYSTEM', Job::WORKER_DEFAULT, 'virus_scan', ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'], 'Scanning...');
$processId = CakeResque::enqueue(
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'AdminShell',
array('scanAttachment', $type, $attribute['id'], $jobId),
true
'virus_scan',
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'Scanning...'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'scanAttachment',
$type,
$attribute['id'],
$jobId
],
true,
$jobId
);
$job->saveField('process_id', $processId);
}
}

1344
app/Model/Attribute.php
View File

File diff suppressed because it is too large Load Diff

48
app/Model/AttributeTag.php
View File

@ -342,42 +342,40 @@ class AttributeTag extends AppModel
return $allClusters;
}
public function extractAttributeTagsNameFromEvent(&$event, $to_extract='both')
/**
* @param array $event
* @return array|array[]
*/
public function extractAttributeTagsNameFromEvent(array $event)
{
$attribute_tags_name = array('tags' => array(), 'clusters' => array());
foreach ($event['Attribute'] as $i => $attribute) {
if ($to_extract == 'tags' || $to_extract == 'both') {
foreach ($attribute['AttributeTag'] as $tag) {
$attribute_tags_name['tags'][] = $tag['Tag']['name'];
}
$extractedTags = [];
$extractedClusters = [];
foreach ($event['Attribute'] as $attribute) {
foreach ($attribute['AttributeTag'] as $tag) {
$extractedTags[$tag['Tag']['id']] = $tag['Tag']['name'];
}
if ($to_extract == 'clusters' || $to_extract == 'both') {
foreach ($attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$attribute_tags_name['clusters'][] = $cluster['tag_name'];
}
foreach ($attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$extractedClusters[$cluster['tag_id']] = $cluster['tag_name'];
}
}
}
foreach ($event['Object'] as $i => $object) {
foreach ($event['Object'] as $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $j => $object_attribute) {
if ($to_extract == 'tags' || $to_extract == 'both') {
foreach ($object_attribute['AttributeTag'] as $tag) {
$attribute_tags_name['tags'][] = $tag['Tag']['name'];
}
foreach ($object['Attribute'] as $object_attribute) {
foreach ($object_attribute['AttributeTag'] as $tag) {
$extractedTags[$tag['Tag']['id']] = $tag['Tag']['name'];
}
if ($to_extract == 'clusters' || $to_extract == 'both') {
foreach ($object_attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$attribute_tags_name['clusters'][] = $cluster['tag_name'];
}
foreach ($object_attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$extractedClusters[$cluster['tag_id']] = $cluster['tag_name'];
}
}
}
}
}
$attribute_tags_name['tags'] = array_diff_key($attribute_tags_name['tags'], $attribute_tags_name['clusters']); // de-dup if needed.
return $attribute_tags_name;
$extractedTags = array_diff_key($extractedTags, $extractedClusters); // de-dup if needed.
return ['tags' => $extractedTags, 'clusters' => $extractedClusters];
}
}

76
app/Model/AuditLog.php
View File

@ -41,6 +41,15 @@ class AuditLog extends AppModel
/** @var bool */
private $compressionEnabled;
/** @var bool */
private $pubToZmq;
/** @var bool */
private $elasticLogging;
/** @var bool */
private $logClientIp;
/**
* Null when not defined, false when not enabled
* @var Syslog|null|false
@ -72,19 +81,22 @@ class AuditLog extends AppModel
{
parent::__construct($id, $table, $ds);
$this->compressionEnabled = Configure::read('MISP.log_new_audit_compress') && function_exists('brotli_compress');
$this->pubToZmq = $this->pubToZmq('audit');
$this->elasticLogging = Configure::read('Plugin.ElasticSearch_logging_enable');
$this->logClientIp = Configure::read('MISP.log_client_ip');
}
public function afterFind($results, $primary = false)
{
foreach ($results as $key => $result) {
foreach ($results as &$result) {
if (isset($result['AuditLog']['ip'])) {
$results[$key]['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']);
$result['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']);
}
if (isset($result['AuditLog']['change']) && $result['AuditLog']['change']) {
$results[$key]['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']);
$result['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']);
}
if (isset($result['AuditLog']['action']) && isset($result['AuditLog']['model']) && isset($result['AuditLog']['model_id'])) {
$results[$key]['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']);
$result['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']);
}
}
return $results;
@ -119,13 +131,17 @@ class AuditLog extends AppModel
if (in_array($auditLog['model'], ['Attribute', 'Object', 'ShadowAttribute'], true)) {
$modelName = $auditLog['model'] === 'ShadowAttribute' ? 'Proposal' : $auditLog['model'];
$title = __('%s from Event #%s', $modelName, $auditLog['event_id']);
} else {
$title = "{$auditLog['model']} #{$auditLog['model_id']}";
}
if (isset($auditLog['model_title']) && $auditLog['model_title']) {
$title .= ": {$auditLog['model_title']}";
if (isset($title)) {
$title .= ": {$auditLog['model_title']}";
return $title;
} else {
return $auditLog['model_title'];
}
}
return $title;
return '';
}
/**
@ -160,51 +176,52 @@ class AuditLog extends AppModel
public function beforeSave($options = array())
{
if (!isset($this->data['AuditLog']['ip']) && Configure::read('MISP.log_client_ip')) {
$auditLog = &$this->data['AuditLog'];
if (!isset($auditLog['ip']) && $this->logClientIp) {
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
if (isset($_SERVER[$ipHeader])) {
$this->data['AuditLog']['ip'] = inet_pton($_SERVER[$ipHeader]);
$auditLog['ip'] = inet_pton($_SERVER[$ipHeader]); // convert to binary form
}
}
if (!isset($this->data['AuditLog']['user_id'])) {
$this->data['AuditLog']['user_id'] = $this->userInfo()['id'];
if (!isset($auditLog['user_id'])) {
$auditLog['user_id'] = $this->userInfo()['id'];
}
if (!isset($this->data['AuditLog']['org_id'])) {
$this->data['AuditLog']['org_id'] = $this->userInfo()['org_id'];
if (!isset($auditLog['org_id'])) {
$auditLog['org_id'] = $this->userInfo()['org_id'];
}
if (!isset($this->data['AuditLog']['request_type'])) {
$this->data['AuditLog']['request_type'] = $this->userInfo()['request_type'];
if (!isset($auditLog['request_type'])) {
$auditLog['request_type'] = $this->userInfo()['request_type'];
}
if (!isset($this->data['AuditLog']['authkey_id'])) {
$this->data['AuditLog']['authkey_id'] = $this->userInfo()['authkey_id'];
if (!isset($auditLog['authkey_id'])) {
$auditLog['authkey_id'] = $this->userInfo()['authkey_id'];
}
if (!isset($this->data['AuditLog']['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$this->data['AuditLog']['request_id'] = $_SERVER['HTTP_X_REQUEST_ID'];
if (!isset($auditLog['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$auditLog['request_id'] = $_SERVER['HTTP_X_REQUEST_ID'];
}
// Truncate request_id
if (isset($this->data['AuditLog']['request_id']) && strlen($this->data['AuditLog']['request_id']) > 255) {
$this->data['AuditLog']['request_id'] = substr($this->data['AuditLog']['request_id'], 0, 255);
if (isset($auditLog['request_id']) && strlen($auditLog['request_id']) > 255) {
$auditLog['request_id'] = substr($auditLog['request_id'], 0, 255);
}
// Truncate model title
if (isset($this->data['AuditLog']['model_title']) && mb_strlen($this->data['AuditLog']['model_title']) > 255) {
$this->data['AuditLog']['model_title'] = mb_substr($this->data['AuditLog']['model_title'], 0, 252) . '...';
if (isset($auditLog['model_title']) && mb_strlen($auditLog['model_title']) > 255) {
$auditLog['model_title'] = mb_substr($auditLog['model_title'], 0, 252) . '...';
}
$this->logData($this->data);
if (isset($this->data['AuditLog']['change'])) {
$change = json_encode($this->data['AuditLog']['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
if (isset($auditLog['change'])) {
$change = json_encode($auditLog['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) {
$change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT);
}
$this->data['AuditLog']['change'] = $change;
$auditLog['change'] = $change;
}
}
@ -214,14 +231,14 @@ class AuditLog extends AppModel
*/
private function logData(array $data)
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) {
if ($this->pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->publish($data, 'audit', 'log');
}
$this->publishKafkaNotification('audit', $data, 'log');
if (Configure::read('Plugin.ElasticSearch_logging_enable')) {
if ($this->elasticLogging) {
// send off our logs to distributed /dev/null
$logIndex = Configure::read("Plugin.ElasticSearch_log_index");
$elasticSearchClient = $this->getElasticSearchTool();
@ -344,6 +361,7 @@ class AuditLog extends AppModel
'conditions' => $conditions,
'group' => ['Date'],
'order' => ['Date'],
'callbacks' => false,
]);
} elseif ($dataSource === 'Database/Postgres') {
if (!empty($conditions['org_id'])) {

84
app/Model/Behavior/AuditLogBehavior.php
View File

@ -3,9 +3,6 @@ App::uses('AuditLog', 'Model');
class AuditLogBehavior extends ModelBehavior
{
/** @var array */
private $config;
/** @var array|null */
private $old;
@ -27,6 +24,8 @@ class AuditLogBehavior extends ModelBehavior
'last_login' => true, // User
'newsread' => true, // User
'proposal_email_lock' => true, // Event
'enable_password' => true,
'confirm_password' => true
];
private $modelInfo = [
@ -42,6 +41,7 @@ class AuditLogBehavior extends ModelBehavior
'TagCollection' => 'name',
'Taxonomy' => 'namespace',
'Organisation' => 'name',
'SystemSetting' => 'setting',
'AdminSetting' => 'setting',
'UserSetting' => 'setting',
'Galaxy' => 'name',
@ -57,7 +57,6 @@ class AuditLogBehavior extends ModelBehavior
public function setup(Model $model, $config = [])
{
$this->config = $config;
// Generate model info for attribute and proposals
$attributeInfo = function (array $new, array $old) {
$category = isset($new['category']) ? $new['category'] : $old['category'];
@ -81,11 +80,33 @@ class AuditLogBehavior extends ModelBehavior
if (!$this->enabled) {
return true;
}
// Do not fetch old version when just few fields will be fetched
$fieldToFetch = [];
if (!empty($options['fieldList'])) {
foreach ($options['fieldList'] as $field) {
if (!isset($this->skipFields[$field])) {
$fieldToFetch[] = $field;
}
}
// For objects, that are assigned to event, we need to know event ID. So if data to save doesn't contain
// that ID, we need to fetch it from database.
if (isset($model->schema()['event_id']) && empty($model->data[$model->alias]['event_id']) && !in_array('event_id', $fieldToFetch, true)) {
$fieldToFetch[] = 'event_id';
}
if (empty($fieldToFetch)) {
$this->old = null;
return true;
}
}
if ($model->id) {
$this->old = $model->find('first', [
'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id],
'recursive' => -1,
'callbacks' => false,
'fields' => $fieldToFetch,
]);
} else {
$this->old = null;
@ -99,36 +120,31 @@ class AuditLogBehavior extends ModelBehavior
return;
}
if ($model->id) {
$id = $model->id;
} else if ($model->insertId) {
$id = $model->insertId;
} else {
$id = null;
}
$id = $model->id ?: 0;
$data = $model->data[$model->alias];
if ($created) {
$action = AuditLog::ACTION_ADD;
} else {
$action = AuditLog::ACTION_EDIT;
if (isset($model->data[$model->alias]['deleted'])) {
if ($model->data[$model->alias]['deleted']) {
if (isset($data['deleted'])) {
if ($data['deleted']) {
$action = AuditLog::ACTION_SOFT_DELETE;
} else if (!$model->data[$model->alias]['deleted'] && $this->old[$model->alias]['deleted']) {
} else if (isset($this->old[$model->alias]['deleted']) && $this->old[$model->alias]['deleted']) {
$action = AuditLog::ACTION_UNDELETE;
}
}
}
$changedFields = $this->changedFields($model, isset($options['fieldList']) ? $options['fieldList'] : null);
$changedFields = $this->changedFields($model, $options['fieldList']);
if (empty($changedFields)) {
return;
}
if ($model->name === 'Event') {
$eventId = $id;
} else if (isset($model->data[$model->alias]['event_id'])) {
$eventId = $model->data[$model->alias]['event_id'];
} else if (isset($data['event_id'])) {
$eventId = $data['event_id'];
} else if (isset($this->old[$model->alias]['event_id'])) {
$eventId = $this->old[$model->alias]['event_id'];
} else {
@ -139,9 +155,9 @@ class AuditLogBehavior extends ModelBehavior
if (isset($this->modelInfo[$model->name])) {
$modelTitleField = $this->modelInfo[$model->name];
if (is_callable($modelTitleField)) {
$modelTitle = $modelTitleField($model->data[$model->alias], isset($this->old[$model->alias]) ? $this->old[$model->alias] : []);
} else if (isset($model->data[$model->alias][$modelTitleField])) {
$modelTitle = $model->data[$model->alias][$modelTitleField];
$modelTitle = $modelTitleField($data, isset($this->old[$model->alias]) ? $this->old[$model->alias] : []);
} else if (isset($data[$modelTitleField])) {
$modelTitle = $data[$modelTitleField];
} else if ($this->old[$model->alias][$modelTitleField]) {
$modelTitle = $this->old[$model->alias][$modelTitleField];
}
@ -150,9 +166,9 @@ class AuditLogBehavior extends ModelBehavior
$modelName = $model->name === 'MispObject' ? 'Object' : $model->name;
if ($modelName === 'AttributeTag' || $modelName === 'EventTag') {
$isLocal = isset($model->data[$model->alias]['local']) ? $model->data[$model->alias]['local'] : false;
$isLocal = isset($data['local']) ? $data['local'] : false;
$action = $isLocal ? AuditLog::ACTION_TAG_LOCAL : AuditLog::ACTION_TAG;
$tagInfo = $this->getTagInfo($model, $model->data[$model->alias]['tag_id']);
$tagInfo = $this->getTagInfo($model, $data['tag_id']);
if ($tagInfo) {
$modelTitle = $tagInfo['tag_name'];
if ($tagInfo['is_galaxy']) {
@ -162,26 +178,26 @@ class AuditLogBehavior extends ModelBehavior
}
}
}
$id = $modelName === 'AttributeTag' ? $model->data[$model->alias]['attribute_id'] : $model->data[$model->alias]['event_id'];
$id = $modelName === 'AttributeTag' ? $data['attribute_id'] : $data['event_id'];
$modelName = $modelName === 'AttributeTag' ? 'Attribute' : 'Event';
}
if ($modelName === 'Event') {
} else if ($modelName === 'Event') {
if (isset($changedFields['published'][1]) && $changedFields['published'][1]) {
$action = AuditLog::ACTION_PUBLISH;
} else if (isset($changedFields['sighting_timestamp'][1]) && $changedFields['sighting_timestamp'][1]) {
$action = AuditLog::ACTION_PUBLISH_SIGHTINGS;
}
} else if ($modelName === 'SystemSetting') {
$id = 0;
}
$this->auditLog()->insert([
$this->auditLog()->insert(['AuditLog' => [
'action' => $action,
'model' => $modelName,
'model_id' => $id,
'model_title' => $modelTitle,
'event_id' => $eventId,
'change' => $changedFields,
]);
]]);
}
public function beforeDelete(Model $model, $cascade = true)
@ -240,16 +256,18 @@ class AuditLogBehavior extends ModelBehavior
}
$id = $modelName === 'AttributeTag' ? $model->data[$model->alias]['attribute_id'] : $model->data[$model->alias]['event_id'];
$modelName = $modelName === 'AttributeTag' ? 'Attribute' : 'Event';
} else if ($modelName === 'SystemSetting') {
$id = 0;
}
$this->auditLog()->insert([
$this->auditLog()->insert(['AuditLog' => [
'action' => $action,
'model' => $modelName,
'model_id' => $id,
'model_title' => $modelTitle,
'event_id' => $eventId,
'change' => $this->changedFields($model),
]);
]]);
}
/**
@ -263,6 +281,7 @@ class AuditLogBehavior extends ModelBehavior
'conditions' => ['Tag.id' => $tagId],
'recursive' => -1,
'fields' => ['Tag.name', 'Tag.is_galaxy'],
'callbacks' => false, // disable Tag::afterFind callback
]);
if (empty($tag)) {
return null;
@ -299,6 +318,7 @@ class AuditLogBehavior extends ModelBehavior
{
$dbFields = $model->schema();
$changedFields = [];
$hasPrimaryField = isset($model->data[$model->alias][$model->primaryKey]);
foreach ($model->data[$model->alias] as $key => $value) {
if (isset($this->skipFields[$key])) {
continue;
@ -310,7 +330,7 @@ class AuditLogBehavior extends ModelBehavior
continue;
}
if (isset($model->data[$model->alias][$model->primaryKey]) && isset($this->old[$model->alias][$key])) {
if ($hasPrimaryField && isset($this->old[$model->alias][$key])) {
$old = $this->old[$model->alias][$key];
} else {
$old = null;
@ -335,7 +355,7 @@ class AuditLogBehavior extends ModelBehavior
continue;
}
if ($key === 'password' || $key === 'authkey') {
if ($key === 'password' || $key === 'authkey' || ($key === 'value' && $model->name === 'SystemSetting' && SystemSetting::isSensitive($model->data[$model->alias]['setting']))) {
$value = '*****';
if ($old !== null) {
$old = $value;

52
app/Model/Cerebrate.php
View File

@ -1,6 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('EncryptedValue', 'Tools');
class Cerebrate extends AppModel
{
@ -16,12 +16,23 @@ class Cerebrate extends AppModel
public $belongsTo = array(
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id'
'className' => 'Organisation',
'foreignKey' => 'org_id'
)
);
public function queryInstance($options) {
public function beforeSave($options = array())
{
$cerebrate = &$this->data['Server'];
// Encrypt authkey if plain key provided and encryption is enabled
if (!empty($cerebrate['authkey']) && strlen($cerebrate['authkey']) === 40) {
$cerebrate['authkey'] = EncryptedValue::encryptIfEnabled($cerebrate['authkey']);
}
return true;
}
public function queryInstance($options)
{
$url = $options['cerebrate']['Cerebrate']['url'] . $options['path'];
$url_params = [];
@ -413,4 +424,37 @@ class Cerebrate extends AppModel
}
return __('The retrieved data isn\'t a valid sharing group.');
}
/**
* @param string|null $old Old (or current) encryption key.
* @param string|null $new New encryption key. If empty, encrypted values will be decrypted.
* @throws Exception
*/
public function reencryptAuthKeys($old, $new)
{
$cerebrates = $this->find('list', [
'fields' => ['Cerebrate.id', 'Cerebrate.authkey'],
]);
$toSave = [];
foreach ($cerebrates as $id => $authkey) {
if (EncryptedValue::isEncrypted($authkey)) {
try {
$authkey = BetterSecurity::decrypt(substr($authkey, 2), $old);
} catch (Exception $e) {
throw new Exception("Could not decrypt auth key for Cerebrate #$id", 0, $e);
}
}
if (!empty($new)) {
$authkey = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($authkey, $new);
}
$toSave[] = ['Cerebrate' => [
'id' => $id,
'authkey' => $authkey,
]];
}
if (empty($toSave)) {
return true;
}
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
}
}

82
app/Model/Correlation.php
View File

@ -43,29 +43,29 @@ class Correlation extends AppModel
public function correlateValueRouter($value)
{
if (Configure::read('MISP.background_jobs')) {
if (empty($this->Job)) {
$this->Job = ClassRegistry::init('Job');
}
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'correlateValue',
'job_input' => $value,
'status' => 0,
'retries' => 0,
'org_id' => 0,
'org' => 0,
'message' => 'Recorrelating',
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'correlateValue',
$value,
'Recorrelating'
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'EventShell',
['correlateValue', $value, $jobId],
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'correlateValue',
$value,
$jobId
],
true,
$jobId
);
$this->Job->saveField('process_id', $process_id);
return true;
} else {
return $this->correlateValue($value);
@ -590,29 +590,27 @@ class Correlation extends AppModel
public function generateTopCorrelationsRouter()
{
if (Configure::read('MISP.background_jobs')) {
if (empty($this->Job)) {
$this->Job = ClassRegistry::init('Job');
}
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'generateTopCorrelations',
'job_input' => '',
'status' => 0,
'retries' => 0,
'org_id' => 0,
'org' => 0,
'message' => 'Starting generation of top correlations.',
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generateTopCorrelations',
'',
'Starting generation of top correlations.'
);
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'EventShell',
['generateTopCorrelations', $jobId],
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'generateTopCorrelations',
$jobId
],
true,
$jobId
);
$this->Job->saveField('process_id', $process_id);
return $jobId;
} else {
return $this->generateTopCorrelations();

39
app/Model/CorrelationExclusion.php
View File

@ -67,27 +67,26 @@ class CorrelationExclusion extends AppModel
public function cleanRouter($user)
{
if (Configure::read('MISP.background_jobs')) {
$this->Job = ClassRegistry::init('Job');
$this->Job->create();
$data = [
'worker' => 'default',
'job_type' => 'clean_correlation_exclusions',
'job_input' => '',
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
'message' => __('Cleaning up excluded correlations.'),
];
$this->Job->save($data);
$jobId = $this->Job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
['cleanExcludedCorrelations', $jobId],
true
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
$user,
Job::WORKER_DEFAULT,
'clean_correlation_exclusions',
'',
__('Cleaning up excluded correlations.')
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'cleanExcludedCorrelations',
$jobId
],
true,
$jobId
);
$this->Job->saveField('process_id', $process_id);
$message = __('Cleanup queued for background execution.');
} else {
$this->clean();
}

996
app/Model/Event.php
View File

File diff suppressed because it is too large Load Diff

12
app/Model/EventGraph.php
View File

@ -30,7 +30,7 @@ class EventGraph extends AppModel
public $validate = array(
'network_json' => array(
'rule' => array('isValidJson'),
'rule' => 'valueIsJson',
'message' => 'The provided eventGraph is not a valid json format',
'required' => true,
),
@ -44,16 +44,6 @@ class EventGraph extends AppModel
return true;
}
public function isValidJson($fields)
{
$text = $fields['network_json'];
$check = json_decode($text);
if ($check === null) {
return false;
}
return true;
}
public function getPictureData($eventGraph)
{
$b64 = str_replace('data:image/png;base64,', '', $eventGraph['EventGraph']['preview_img']);

44
app/Model/EventTag.php
View File

@ -141,6 +141,50 @@ class EventTag extends AppModel
return false;
}
/**
* Find all of the event Ids that belong to the accepted tags and the rejected tags
* @param array $accept
* @param array $reject
* @return array[]
*/
public function fetchEventTagIds(array $accept = array(), array $reject = array())
{
$acceptIds = array();
$rejectIds = array();
if (!empty($accept)) {
$acceptIds = $this->findEventIdsByTagNames($accept);
if (empty($acceptIds)) {
$acceptIds = [-1];
}
}
if (!empty($reject)) {
$rejectIds = $this->findEventIdsByTagNames($reject);
}
return array($acceptIds, $rejectIds);
}
/**
* @param array $tagIdsOrNames
* @return array|int|null
*/
private function findEventIdsByTagNames(array $tagIdsOrNames)
{
$conditions = [];
foreach ($tagIdsOrNames as $tagIdOrName) {
if (is_numeric($tagIdOrName)) {
$conditions[] = array('Tag.id' => $tagIdOrName);
} else {
$conditions[] = array('LOWER(Tag.name)' => mb_strtolower($tagIdOrName));
}
}
return $this->find('column', array(
'recursive' => -1,
'contain' => 'Tag',
'conditions' => ['OR' => $conditions],
'fields' => ['EventTag.event_id'],
));
}
public function getSortedTagList($context = false)
{
$conditions = array();

171
app/Model/Feed.php
View File

@ -2,7 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
App::uses('FileAccessTool', 'Tools');
App::uses('AttributeValidationTool', 'Tools');
class Feed extends AppModel
{
@ -190,7 +190,7 @@ class Feed extends AppModel
/**
* @param array $feed
* @param HttpSocket|null $HttpSocket Null can be for local feed
* @return Generator
* @return Generator<string>
* @throws Exception
*/
public function getCache(array $feed, HttpSocket $HttpSocket = null)
@ -355,13 +355,10 @@ class Feed extends AppModel
* @param array $feed
* @param HttpSocket|null $HttpSocket Null can be for local feed
* @param string $type
* @param int|string $page
* @param int $limit
* @param array $params
* @return array|bool
* @throws Exception
*/
public function getFreetextFeed($feed, HttpSocket $HttpSocket = null, $type = 'freetext', $page = 1, $limit = 60, &$params = array())
public function getFreetextFeed($feed, HttpSocket $HttpSocket = null, $type = 'freetext')
{
if ($this->isFeedLocal($feed)) {
$feedUrl = $feed['Feed']['url'];
@ -386,18 +383,9 @@ class Feed extends AppModel
}
$resultArray = $complexTypeTool->checkComplexRouter($data, $type, $settings);
$this->Attribute = ClassRegistry::init('Attribute');
foreach ($resultArray as $key => $value) {
$resultArray[$key]['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category'];
}
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
$params = $customPagination->createPaginationRules($resultArray, array('page' => $page, 'limit' => $limit), 'Feed', $sort = false);
if (!empty($page) && $page != 'all') {
$start = ($page - 1) * $limit;
if ($start > count($resultArray)) {
return false;
}
$resultArray = array_slice($resultArray, $start, $limit);
$typeDefinitions = $this->Attribute->typeDefinitions;
foreach ($resultArray as &$value) {
$value['category'] = $typeDefinitions[$value['default_type']]['default_category'];
}
return $resultArray;
}
@ -478,7 +466,7 @@ class Feed extends AppModel
}
$compositeTypes = $this->Attribute->getCompositeTypes();
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
$hashTable = [];
$redisResultToAttributePosition = [];
@ -550,7 +538,7 @@ class Feed extends AppModel
foreach ($sources as $source) {
$sourceId = $source[$scope]['id'];
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
foreach ($hitIds as $k) {
$redis->sismember($cachePrefix . $sourceId, $hashTable[$k]);
}
@ -574,7 +562,7 @@ class Feed extends AppModel
// Append also exact MISP feed or server event UUID
// TODO: This can be optimised in future to do that in one pass
if ($sourceHasHit && ($scope === 'Server' || $source[$scope]['source_format'] === 'misp')) {
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
$eventUuidHitPosition = [];
foreach ($hitIds as $sourceHitPos => $k) {
if ($sourceHits[$sourceHitPos]) {
@ -648,7 +636,7 @@ class Feed extends AppModel
try {
$redis = $this->setupRedisWithException();
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
$cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
foreach ($sources as $source) {
$pipe->exists($cachePrefix . $source[$scope]['id']);
@ -1055,19 +1043,32 @@ class Feed extends AppModel
return $success;
}
/**
* @param int $feedId
* @param array $user
* @param int|false $jobId
* @return array|bool
* @throws Exception
*/
public function downloadFromFeedInitiator($feedId, $user, $jobId = false)
{
$this->id = $feedId;
$this->read();
if (isset($this->data['Feed']['settings']) && !empty($this->data['Feed']['settings'])) {
$this->data['Feed']['settings'] = json_decode($this->data['Feed']['settings'], true);
$feed = $this->find('first', array(
'conditions' => ['Feed.id' => $feedId],
'recursive' => -1,
));
if (empty($feed)) {
throw new Exception("Feed with ID $feedId not found.");
}
$HttpSocket = $this->isFeedLocal($this->data) ? null : $this->__setupHttpSocket();
if ($this->data['Feed']['source_format'] === 'misp') {
if (!empty($feed['Feed']['settings'])) {
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);
}
$HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket();
if ($feed['Feed']['source_format'] === 'misp') {
$this->jobProgress($jobId, 'Fetching event manifest.');
try {
$actions = $this->getNewEventUuids($this->data, $HttpSocket);
$actions = $this->getNewEventUuids($feed, $HttpSocket);
} catch (Exception $e) {
$this->logException("Could not get new event uuids for feed $feedId.", $e);
$this->jobProgress($jobId, 'Could not fetch event manifest. See error log for more details.');
@ -1080,12 +1081,12 @@ class Feed extends AppModel
$total = count($actions['add']) + count($actions['edit']);
$this->jobProgress($jobId, __("Fetching %s events.", $total));
$result = $this->downloadFromFeed($actions, $this->data, $HttpSocket, $user, $jobId);
$this->__cleanupFile($this->data, '/manifest.json');
$result = $this->downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId);
$this->__cleanupFile($feed, '/manifest.json');
} else {
$this->jobProgress($jobId, 'Fetching data.');
try {
$temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all');
$temp = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']);
} catch (Exception $e) {
$this->logException("Could not get freetext feed $feedId", $e);
$this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.');
@ -1105,17 +1106,18 @@ class Feed extends AppModel
'to_ids' => $value['to_ids']
);
}
unset($temp);
$this->jobProgress($jobId, 'Saving data.', 50);
try {
$result = $this->saveFreetextFeedData($this->data, $data, $user, $jobId);
$result = $this->saveFreetextFeedData($feed, $data, $user, $jobId);
} catch (Exception $e) {
$this->logException("Could not save freetext feed data for feed $feedId.", $e);
return false;
}
$this->__cleanupFile($this->data, '');
$this->__cleanupFile($feed, '');
}
return $result;
}
@ -1201,7 +1203,7 @@ class Feed extends AppModel
// Because some types can be saved in modified version (for example, IPv6 address is convert to compressed
// format, we should also check if current event contains modified value.
$modifiedValue = $this->Event->Attribute->modifyBeforeValidation($dataPoint['type'], $dataPoint['value']);
$modifiedValue = AttributeValidationTool::modifyBeforeValidation($dataPoint['type'], $dataPoint['value']);
if (isset($existsAttributesValueToId[$modifiedValue])) {
unset($data[$k]);
unset($existsAttributesValueToId[$modifiedValue]);
@ -1235,8 +1237,8 @@ class Feed extends AppModel
continue;
}
$data[$key]['event_id'] = $event['Event']['id'];
$data[$key]['distribution'] = $feed['Feed']['distribution'];
$data[$key]['sharing_group_id'] = $feed['Feed']['sharing_group_id'];
$data[$key]['distribution'] = 5;
$data[$key]['sharing_group_id'] = 0;
$data[$key]['to_ids'] = $feed['Feed']['override_ids'] ? 0 : $value['to_ids'];
$uniqueValues[$value['value']] = true;
}
@ -1318,7 +1320,7 @@ class Feed extends AppModel
return $feeds;
}
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
foreach ($feeds as $feed) {
$pipe->get('misp:feed_cache_timestamp:' . $feed['Feed']['id']);
}
@ -1329,14 +1331,28 @@ class Feed extends AppModel
return $feeds;
}
/**
* @param array $feed
* @param Redis $redis
* @param int|false $jobId
* @return bool
*/
private function __cacheFeed($feed, $redis, $jobId = false)
{
$HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket();
if ($feed['Feed']['source_format'] === 'misp') {
return $this->__cacheMISPFeed($feed, $redis, $HttpSocket, $jobId);
$result = true;
if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) {
$result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId);
}
} else {
return $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId);
$result = $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId);
}
if ($result) {
$redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time());
}
return $result;
}
/**
@ -1353,7 +1369,7 @@ class Feed extends AppModel
$this->jobProgress($jobId, __("Feed %s: Fetching.", $feedId));
try {
$values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all');
$values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']);
} catch (Exception $e) {
$this->logException("Could not get freetext feed $feedId", $e);
$this->jobProgress($jobId, __('Could not fetch freetext feed %s. See error log for more details.', $feedId));
@ -1365,7 +1381,7 @@ class Feed extends AppModel
$redis->del('misp:feed_cache:' . $feedId);
foreach (array_chunk($md5Values, 5000) as $k => $chunk) {
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
if (method_exists($redis, 'sAddArray')) {
$redis->sAddArray('misp:feed_cache:' . $feedId, $chunk);
$redis->sAddArray('misp:feed_cache:combined', $chunk);
@ -1378,10 +1394,16 @@ class Feed extends AppModel
$pipe->exec();
$this->jobProgress($jobId, __('Feed %s: %s/%s values cached.', $feedId, $k * 5000, count($md5Values)));
}
$redis->set('misp:feed_cache_timestamp:' . $feedId, time());
return true;
}
/**
* @param array $feed
* @param Redis $redis
* @param HttpSocket|null $HttpSocket
* @param false $jobId
* @return bool
*/
private function __cacheMISPFeedTraditional($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
{
$feedId = $feed['Feed']['id'];
@ -1395,6 +1417,7 @@ class Feed extends AppModel
$redis->del('misp:feed_cache:' . $feedId);
$k = 0;
$this->Attribute = ClassRegistry::init('Attribute');
foreach ($manifest as $uuid => $event) {
try {
$event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
@ -1404,11 +1427,10 @@ class Feed extends AppModel
}
if (!empty($event['Event']['Attribute'])) {
$this->Attribute = ClassRegistry::init('Attribute');
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
foreach ($event['Event']['Attribute'] as $attribute) {
if (!in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES, true)) {
if (in_array($attribute['type'], $this->Attribute->getCompositeTypes())) {
if (in_array($attribute['type'], $this->Attribute->getCompositeTypes(), true)) {
$value = explode('|', $attribute['value']);
if (in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true)) {
unset($value[1]);
@ -1436,6 +1458,13 @@ class Feed extends AppModel
return true;
}
/**
* @param array $feed
* @param Redis $redis
* @param HttpSocket|null $HttpSocket
* @param false $jobId
* @return bool
*/
private function __cacheMISPFeedCache($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
{
$feedId = $feed['Feed']['id'];
@ -1447,7 +1476,7 @@ class Feed extends AppModel
return false;
}
$pipe = $redis->multi(Redis::PIPELINE);
$pipe = $redis->pipeline();
$redis->del('misp:feed_cache:' . $feedId);
foreach ($cache as $v) {
list($hash, $eventUuid) = $v;
@ -1460,18 +1489,6 @@ class Feed extends AppModel
return true;
}
private function __cacheMISPFeed($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
{
$result = true;
if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) {
$result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId);
}
if ($result) {
$redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time());
}
return $result;
}
public function compareFeeds($id = false)
{
$redis = $this->setupRedis();
@ -1747,19 +1764,12 @@ class Feed extends AppModel
{
$hits = array();
$this->Server = ClassRegistry::init('Server');
$result['Server'] = $this->Server->find('all', array(
'conditions' => array(
'caching_enabled' => 1
),
'recursive' => -1,
'fields' => array('Server.id', 'Server.name', 'Server.url')
));
$redis = $this->setupRedis();
$is_array = true;
if (!is_array($value)) {
$is_array = false;
if (empty($value)) {
// old behaviour allowd for empty values to return all data
// old behaviour allowed for empty values to return all data
$value = [false];
} else {
$value = [$value];
@ -1821,7 +1831,6 @@ class Feed extends AppModel
}
}
if ($v === false || $redis->sismember('misp:server_cache:combined', md5($v))) {
$this->Server = ClassRegistry::init('Server');
$servers = $this->Server->find('all', array(
'conditions' => array(
'caching_enabled' => 1
@ -1932,16 +1941,14 @@ class Feed extends AppModel
$contentType = $response->getHeader('Content-Type');
if ($contentType === 'application/zip') {
$zipFile = new File($this->tempFileName());
$zipFile->write($response->body);
$zipFile->close();
$zipFilePath = FileAccessTool::writeToTempFile($response->body);
try {
$response->body = $this->unzipFirstFile($zipFile);
$response->body = $this->unzipFirstFile($zipFilePath);
} catch (Exception $e) {
throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}");
} finally {
$zipFile->delete();
FileAccessTool::deleteFile($zipFilePath);
}
}
@ -2047,18 +2054,18 @@ class Feed extends AppModel
}
/**
* @param File $zipFile
* @param string $zipFile
* @return string Uncompressed data
* @throws Exception
*/
private function unzipFirstFile(File $zipFile)
private function unzipFirstFile($zipFile)
{
if (!class_exists('ZipArchive')) {
throw new Exception('ZIP archive decompressing is not supported. ZIP extension is missing in PHP.');
}
$zip = new ZipArchive();
$result = $zip->open($zipFile->pwd());
$result = $zip->open($zipFile);
if ($result !== true) {
$errorCodes = [
ZipArchive::ER_EXISTS => 'file already exists',
@ -2086,18 +2093,12 @@ class Feed extends AppModel
$zip->close();
$destinationFile = $this->tempFileName();
$result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
$destinationFile = FileAccessTool::createTempFile();
$result = copy("zip://$zipFile#$filename", $destinationFile);
if ($result === false) {
throw new Exception("Remote server returns ZIP file, that contains '$filename' file, but this file cannot be extracted.");
}
$unzipped = new File($destinationFile);
$data = $unzipped->read();
if ($data === false) {
throw new Exception("Couldn't read extracted file content.");
}
$unzipped->delete();
return $data;
return FileAccessTool::readAndDelete($destinationFile);
}
}

59
app/Model/Galaxy.php
View File

@ -57,17 +57,14 @@ class Galaxy extends AppModel
/**
* @param bool $force
* @return array Galaxy type => Galaxy ID
* @throws JsonException
* @throws Exception
*/
private function __load_galaxies($force = false)
{
$dir = new Folder(APP . 'files' . DS . 'misp-galaxy' . DS . 'galaxies');
$files = $dir->find('.*\.json');
$files = new GlobIterator(APP . 'files' . DS . 'misp-galaxy' . DS . 'galaxies' . DS . '*.json');
$galaxies = array();
foreach ($files as $file) {
$file = new File($dir->pwd() . DS . $file);
$galaxies[] = $this->jsonDecode($file->read());
$file->close();
$galaxies[] = FileAccessTool::readJsonFromFile($file->getPathname());
}
$existingGalaxies = $this->find('all', array(
'fields' => array('uuid', 'version', 'id', 'icon'),
@ -157,6 +154,10 @@ class Galaxy extends AppModel
$relations = [];
$elements = [];
$this->GalaxyCluster->bulkEntry = true;
// Start transaction
$this->getDataSource()->begin();
foreach ($cluster_package['values'] as $cluster) {
if (empty($cluster['version'])) {
$cluster['version'] = 1;
@ -182,7 +183,12 @@ class Galaxy extends AppModel
$cluster_to_save['published'] = false;
$cluster_to_save['org_id'] = 0;
$cluster_to_save['orgc_id'] = 0;
$result = $this->GalaxyCluster->save($cluster_to_save, false);
// We are already in transaction
$result = $this->GalaxyCluster->save($cluster_to_save, ['atomic' => false, 'validate' => false]);
if (!$result) {
$this->log("Could not save galaxy cluster with UUID {$cluster_to_save['uuid']}.");
continue;
}
$galaxyClusterId = $this->GalaxyCluster->id;
if (isset($cluster['meta'])) {
foreach ($cluster['meta'] as $key => $value) {
@ -206,7 +212,7 @@ class Galaxy extends AppModel
$elements[] = array(
$galaxyClusterId,
$key,
strval($v)
(string)$v
);
}
}
@ -221,24 +227,26 @@ class Galaxy extends AppModel
'referenced_galaxy_cluster_type' => $relation['type'],
'default' => true,
'distribution' => 3,
'tags' => !empty($relation['tags']) ? $relation['tags'] : []
'tags' => $relation['tags'] ?? []
];
}
}
}
// Commit transaction
$this->getDataSource()->commit();
return [$elements, $relations];
}
public function update($force = false)
{
$galaxies = $this->__load_galaxies($force);
$dir = new Folder(APP . 'files' . DS . 'misp-galaxy' . DS . 'clusters');
$files = $dir->find('.*\.json');
$files = new GlobIterator(APP . 'files' . DS . 'misp-galaxy' . DS . 'clusters' . DS . '*.json');
$force = (bool)$force;
$allRelations = [];
foreach ($files as $file) {
$file = new File($dir->pwd() . DS . $file);
$cluster_package = $this->jsonDecode($file->read());
$file->close();
$cluster_package = FileAccessTool::readJsonFromFile($file->getPathname());
if (!isset($galaxies[$cluster_package['type']])) {
continue;
}
@ -253,10 +261,13 @@ class Galaxy extends AppModel
$fields = array('galaxy_cluster_id', 'key', 'value');
$db->insertMulti('galaxy_elements', $fields, $elements);
}
if (!empty($relations)) {
$this->GalaxyCluster->GalaxyClusterRelation->bulkSaveRelations($relations);
}
$allRelations = array_merge($allRelations, $relations);
}
// Save relation as last part when all clusters are created
if (!empty($allRelations)) {
$this->GalaxyCluster->GalaxyClusterRelation->bulkSaveRelations($allRelations);
}
// Probably unnecessary anymore
$this->GalaxyCluster->generateMissingRelations();
return true;
}
@ -348,11 +359,15 @@ class Galaxy extends AppModel
} else {
$local = 0;
}
$cluster_alias = $this->GalaxyCluster->alias;
$galaxy_alias = $this->alias;
$cluster = $this->GalaxyCluster->fetchGalaxyClusters($user, array(
'first' => true,
'conditions' => array('id' => $cluster_id),
'fields' => array('tag_name', 'id', 'value'),
'conditions' => array("${cluster_alias}.id" => $cluster_id),
'contain' => array('Galaxy'),
'fields' => array('tag_name', 'id', 'value', "${galaxy_alias}.local_only"),
), $full=false);
if (empty($cluster)) {
throw new NotFoundException(__('Invalid Galaxy cluster'));
}
@ -368,7 +383,11 @@ class Galaxy extends AppModel
throw new NotFoundException(__('Invalid %s.', $target_type));
}
$target = $target[0];
$tag_id = $this->Tag->captureTag(array('name' => $cluster['GalaxyCluster']['tag_name'], 'colour' => '#0088cc', 'exportable' => 1), $user, true);
$local_only = $cluster['GalaxyCluster']['Galaxy']['local_only'];
if ($local_only && !$local) {
throw new MethodNotAllowedException(__("This Cluster can only be attached in a local scope"));
}
$tag_id = $this->Tag->captureTag(array('name' => $cluster['GalaxyCluster']['tag_name'], 'colour' => '#0088cc', 'exportable' => 1, 'local_only' => $local_only), $user, true);
$existingTag = $this->Tag->$connectorModel->find('first', array('conditions' => array($target_type . '_id' => $target_id, 'tag_id' => $tag_id)));
if (!empty($existingTag)) {
return 'Cluster already attached.';

115
app/Model/GalaxyCluster.php
View File

@ -77,6 +77,7 @@ class GalaxyCluster extends AppModel
private $__clusterCache = array();
private $deletedClusterUUID;
public $bulkEntry = false;
public $hasMany = array(
'GalaxyElement' => array('dependent' => true),
@ -97,20 +98,20 @@ class GalaxyCluster extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (!isset($this->data['GalaxyCluster']['description'])) {
$this->data['GalaxyCluster']['description'] = '';
$cluster = &$this->data['GalaxyCluster'];
if (!isset($cluster['description'])) {
$cluster['description'] = '';
}
if (isset($this->data['GalaxyCluster']['distribution']) && $this->data['GalaxyCluster']['distribution'] != 4) {
$this->data['GalaxyCluster']['sharing_group_id'] = null;
if (isset($cluster['distribution']) && $cluster['distribution'] != 4) {
$cluster['sharing_group_id'] = null;
}
if (!isset($this->data['GalaxyCluster']['published'])) {
$this->data['GalaxyCluster']['published'] = false;
if (!isset($cluster['published'])) {
$cluster['published'] = false;
}
if (!isset($this->data['GalaxyCluster']['authors']) || is_null($this->data['GalaxyCluster']['authors'])) {
$this->data['GalaxyCluster']['authors'] = '';
} elseif (is_array($this->data['GalaxyCluster']['authors'])) {
$this->data['GalaxyCluster']['authors'] = json_encode($this->data['GalaxyCluster']['authors']);
if (!isset($cluster['authors']) || $cluster['authors'] === null) {
$cluster['authors'] = '';
} elseif (is_array($cluster['authors'])) {
$cluster['authors'] = json_encode($cluster['authors']);
}
return true;
}
@ -118,24 +119,24 @@ class GalaxyCluster extends AppModel
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (isset($results[$k][$this->alias]['authors'])) {
if (isset($result[$this->alias]['authors'])) {
$results[$k][$this->alias]['authors'] = json_decode($results[$k][$this->alias]['authors'], true);
}
if (isset($results[$k][$this->alias]['distribution']) && $results[$k][$this->alias]['distribution'] != 4) {
if (isset($result[$this->alias]['distribution']) && $results[$k][$this->alias]['distribution'] != 4) {
unset($results[$k]['SharingGroup']);
}
if (isset($results[$k][$this->alias]['org_id']) && $results[$k][$this->alias]['org_id'] == 0) {
if (isset($result[$this->alias]['org_id']) && $results[$k][$this->alias]['org_id'] == 0) {
if (isset($results[$k]['Org'])) {
$results[$k]['Org'] = $this->Org->genericMISPOrganisation;
}
}
if (isset($results[$k][$this->alias]['orgc_id']) && $results[$k][$this->alias]['orgc_id'] == 0) {
if (isset($result[$this->alias]['orgc_id']) && $results[$k][$this->alias]['orgc_id'] == 0) {
if (isset($results[$k]['Orgc'])) {
$results[$k]['Orgc'] = $this->Org->genericMISPOrganisation;
}
}
if (!empty($results[$k]['GalaxyClusterRelation'])) {
if (!empty($result['GalaxyClusterRelation'])) {
foreach ($results[$k]['GalaxyClusterRelation'] as $i => $relation) {
if (isset($relation['distribution']) && $relation['distribution'] != 4) {
unset($results[$k]['GalaxyClusterRelation'][$i]['SharingGroup']);
@ -149,8 +150,7 @@ class GalaxyCluster extends AppModel
public function afterSave($created, $options = array())
{
// Update all relations IDs that are unknown but saved
parent::afterSave($created, $options);
if (empty($this->bulkEntry)) {
if (!$this->bulkEntry) {
$cluster = $this->data[$this->alias];
$cluster = $this->fetchAndSetUUID($cluster);
$this->GalaxyClusterRelation->updateAll(
@ -457,32 +457,31 @@ class GalaxyCluster extends AppModel
} else {
return false;
}
$this->Event = ClassRegistry::init('Event');
$job_type = 'publish_cluster';
$function = 'publish_galaxy_clusters';
$message = 'Publishing.';
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'prio',
'job_type' => 'publish_galaxy_clusters',
'job_input' => 'Cluster ID: ' . $clusterId,
'status' => 0,
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => $message
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_PRIO,
'publish_galaxy_clusters',
'Cluster ID: ' . $clusterId,
'Publishing.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'EventShell',
array($function, $clusterId, $jobId, $user['id'], $passAlong),
true
return $this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'publish_galaxy_clusters',
$clusterId,
$jobId,
$user['id'],
$passAlong
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
return $process_id;
} else {
$result = $this->publish($cluster, $passAlong=$passAlong);
return $result;
@ -1089,18 +1088,23 @@ class GalaxyCluster extends AppModel
if (isset($options['list']) && $options['list']) {
return $this->find('list', $params);
}
if (isset($options['first']) && $options['first']) {
$clusters = $this->find('first', $params);
} else if (isset($options['count']) && $options['count']) {
$clusterCount = $this->find('count', $params);
return $clusterCount;
return $this->find('count', $params);
} else {
$clusters = $this->find('all', $params);
}
if (empty($clusters)) {
return $clusters;
}
if (isset($options['first']) && $options['first']) {
$clusters = [$clusters];
}
if ($full) {
$clusterIds = array_column(array_column($clusters, 'GalaxyCluster'), 'id');
$targetingClusterRelations = $this->TargetingClusterRelation->fetchRelations($user, array(
@ -1116,11 +1120,15 @@ class GalaxyCluster extends AppModel
$tagsToFetch = Hash::extract($clusters, "{n}.GalaxyClusterRelation.{n}.GalaxyClusterRelationTag.{n}.tag_id");
$tagsToFetch = array_merge($tagsToFetch, Hash::extract($targetingClusterRelations, "GalaxyClusterRelationTag.{n}.tag_id"));
$tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [
'conditions' => ['id' => array_unique($tagsToFetch)],
'recursive' => -1,
]);
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
if (!empty($tagsToFetch)) {
$tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [
'conditions' => ['id' => array_unique($tagsToFetch)],
'recursive' => -1,
]);
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
} else {
$tags = [];
}
foreach ($targetingClusterRelations as $k => $targetingClusterRelation) {
if (!empty($targetingClusterRelation['GalaxyClusterRelationTag'])) {
@ -1142,12 +1150,12 @@ class GalaxyCluster extends AppModel
$sharingGroupData = $this->Event->__cacheSharingGroupData($user, false);
foreach ($clusters as $i => $cluster) {
if (!empty($cluster['GalaxyCluster']['sharing_group_id']) && isset($sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']])) {
$clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']]['SharingGroup'];
$clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']];
}
if (isset($cluster['GalaxyClusterRelation'])) {
foreach ($cluster['GalaxyClusterRelation'] as $j => $relation) {
if (!empty($relation['sharing_group_id']) && isset($sharingGroupData[$relation['sharing_group_id']])) {
$clusters[$i]['GalaxyClusterRelation'][$j]['SharingGroup'] = $sharingGroupData[$relation['sharing_group_id']]['SharingGroup'];
$clusters[$i]['GalaxyClusterRelation'][$j]['SharingGroup'] = $sharingGroupData[$relation['sharing_group_id']];
}
foreach ($relation['GalaxyClusterRelationTag'] as $relationTag) {
if (isset($tags[$relationTag['tag_id']])) {
@ -1166,6 +1174,11 @@ class GalaxyCluster extends AppModel
}
$clusters[$i] = $this->arrangeData($clusters[$i]);
}
if (isset($options['first']) && $options['first']) {
return $clusters[0];
}
return $clusters;
}
@ -1662,7 +1675,7 @@ class GalaxyCluster extends AppModel
{
$this->Server = ClassRegistry::init('Server');
$this->Log = ClassRegistry::init('Log');
$push = $this->Server->checkVersionCompatibility($server, false, $HttpSocket);
$push = $this->Server->checkVersionCompatibility($server, false);
if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) {
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
}

70
app/Model/GalaxyClusterRelation.php
View File

@ -3,6 +3,7 @@ App::uses('AppModel', 'Model');
/**
* @property GalaxyClusterRelationTag $GalaxyClusterRelationTag
* @property GalaxyCluster $TargetCluster
*/
class GalaxyClusterRelation extends AppModel
{
@ -66,10 +67,10 @@ class GalaxyClusterRelation extends AppModel
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (isset($results[$k]['TargetCluster']) && key_exists('id', $results[$k]['TargetCluster']) && is_null($results[$k]['TargetCluster']['id'])) {
if (isset($result['TargetCluster']) && key_exists('id', $result['TargetCluster']) && is_null($result['TargetCluster']['id'])) {
$results[$k]['TargetCluster'] = array();
}
if (isset($results[$k]['GalaxyClusterRelation']['distribution']) && $results[$k]['GalaxyClusterRelation']['distribution'] != 4) {
if (isset($result['GalaxyClusterRelation']['distribution']) && $result['GalaxyClusterRelation']['distribution'] != 4) {
unset($results[$k]['SharingGroup']);
}
}
@ -78,9 +79,9 @@ class GalaxyClusterRelation extends AppModel
public function buildConditions($user, $clusterConditions = true)
{
$this->Event = ClassRegistry::init('Event');
$conditions = [];
if (!$user['Role']['perm_site_admin']) {
$this->Event = ClassRegistry::init('Event');
$alias = $this->alias;
$sgids = $this->Event->cacheSgids($user, true);
$gcOwnerIds = $this->SourceCluster->cacheGalaxyClusterOwnerIDs($user);
@ -346,44 +347,41 @@ class GalaxyClusterRelation extends AppModel
public function bulkSaveRelations(array $relations)
{
if (!isset($this->bulkCache)) {
$this->bulkCache = [
'tag_ids' => []
];
}
// Fetch existing tags Name => ID mapping
$tagNameToId = $this->GalaxyClusterRelationTag->Tag->find('list', [
'fields' => ['Tag.name', 'Tag.id'],
'callbacks' => false,
]);
// Fetch all cluster UUID => ID mapping
$galaxyClusterUuidToId = $this->TargetCluster->find('list', [
'fields' => ['uuid', 'id'],
'callbacks' => false,
]);
$lookupSavedIds = [];
$relationTagsToSave = [];
foreach ($relations as $k => $relation) {
$relations[$k]['referenced_galaxy_cluster_id'] = 0;
$lookupSavedIds[$relation['galaxy_cluster_id']] = true;
foreach ($relations as &$relation) {
if (isset($galaxyClusterUuidToId[$relation['referenced_galaxy_cluster_uuid']])) {
$relation['referenced_galaxy_cluster_id'] = $galaxyClusterUuidToId[$relation['referenced_galaxy_cluster_uuid']];
} else {
$relation['referenced_galaxy_cluster_id'] = 0; // referenced cluster doesn't exists
}
if (!empty($relation['tags'])) {
$lookupSavedIds[$relation['galaxy_cluster_id']] = true;
foreach ($relation['tags'] as $tag) {
if (!isset($this->bulkCache['tag_ids'][$tag])) {
$existingTag = $this->GalaxyClusterRelationTag->Tag->find('first', [
'recursive' => -1,
'fields' => ['Tag.id'],
'conditions' => ['Tag.name' => $tag]
]);
if (empty($existingTag)) {
$this->GalaxyClusterRelationTag->Tag->create();
$this->GalaxyClusterRelationTag->Tag->save([
'name' => $tag,
'colour' => $this->GalaxyClusterRelationTag->Tag->random_color(),
'exportable' => 1,
'org_id' => 0,
'user_id' => 0,
'hide_tag' => Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0
]);
$this->bulkCache['tag_ids'][$tag] = $this->GalaxyClusterRelationTag->Tag->id;
} else {
$this->bulkCache['tag_ids'][$tag] = $existingTag['Tag']['id'];
}
if (!isset($tagNameToId[$tag])) {
$tagNameToId[$tag] = $this->GalaxyClusterRelationTag->Tag->quickAdd($tag);
}
$relationTagsToSave[$relation['galaxy_cluster_uuid']][$relation['referenced_galaxy_cluster_uuid']][] = $this->bulkCache['tag_ids'][$tag];
$relationTagsToSave[$relation['galaxy_cluster_uuid']][$relation['referenced_galaxy_cluster_uuid']][] = $tagNameToId[$tag];
}
}
}
$this->saveAll($relations);
unset($galaxyClusterUuidToId, $tagNameToId);
$this->saveMany($relations, ['validate' => false]); // Some clusters uses invalid UUID :/
// Insert tags
$savedRelations = $this->find('all', [
'recursive' => -1,
'conditions' => ['galaxy_cluster_id' => array_keys($lookupSavedIds)],
@ -393,9 +391,9 @@ class GalaxyClusterRelation extends AppModel
foreach ($savedRelations as $savedRelation) {
$uuid1 = $savedRelation['GalaxyClusterRelation']['galaxy_cluster_uuid'];
$uuid2 = $savedRelation['GalaxyClusterRelation']['referenced_galaxy_cluster_uuid'];
if (!empty($relationTagsToSave[$uuid1][$uuid2])) {
foreach ($relationTagsToSave[$uuid1][$uuid2] as $tag) {
$relation_tags[] = [$savedRelation['GalaxyClusterRelation']['id'], $tag];
if (isset($relationTagsToSave[$uuid1][$uuid2])) {
foreach ($relationTagsToSave[$uuid1][$uuid2] as $tagId) {
$relation_tags[] = [$savedRelation['GalaxyClusterRelation']['id'], $tagId];
}
}
}

3
app/Model/GalaxyClusterRelationTag.php
View File

@ -1,6 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property Tag $Tag
*/
class GalaxyClusterRelationTag extends AppModel
{
public $useTable = 'galaxy_cluster_relation_tags';

62
app/Model/Job.php
View File

@ -10,7 +10,8 @@ class Job extends AppModel
const WORKER_EMAIL = 'email',
WORKER_PRIO = 'prio',
WORKER_DEFAULT = 'default';
WORKER_DEFAULT = 'default',
WORKER_CACHE = 'cache';
public $belongsTo = array(
'Org' => array(
@ -32,42 +33,49 @@ class Job extends AppModel
public function cache($type, $user)
{
$extra = null;
$extra2 = null;
$shell = 'Event';
$this->create();
$data = array(
'worker' => 'cache',
'job_type' => 'cache_' . $type,
'job_input' => $user['Role']['perm_site_admin'] ? 'All events.' : 'Events visible to: ' . $user['Organisation']['name'],
'status' => 0,
'retries' => 0,
'org_id' => $user['Role']['perm_site_admin'] ? 0 : $user['org_id'],
'message' => 'Fetching events.',
$jobId = $this->createJob(
$user,
Job::WORKER_CACHE,
'cache_' . $type,
$user['Role']['perm_site_admin'] ? 'All events.' : 'Events visible to: ' . $user['Organisation']['name'],
'Fetching events.'
);
$this->save($data);
$id = $this->id;
$this->Event = ClassRegistry::init('Event');
if (in_array($type, array_keys($this->Event->export_types)) && $type !== 'bro') {
$process_id = CakeResque::enqueue(
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::CACHE_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'cache',
$shell . 'Shell',
array('cache', $user['id'], $id, $type),
true
$user['id'],
$jobId,
$type
],
true,
$jobId
);
} elseif ($type === 'bro') {
$type = 'bro';
$process_id = CakeResque::enqueue(
'cache',
$shell . 'Shell',
array('cachebro', $user['id'], $id),
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::CACHE_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'cachebro',
$user['id'],
$jobId,
$type
],
true,
$jobId
);
} else {
throw new MethodNotAllowedException('Invalid export type.');
}
$this->saveField('process_id', $process_id);
return $id;
return $jobId;
}
/**

58
app/Model/Log.php
View File

@ -240,6 +240,9 @@ class Log extends AppModel
if ($action === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) {
return null;
}
if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) {
return null;
}
throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors));
}
@ -247,6 +250,22 @@ class Log extends AppModel
return $result;
}
/**
* @param array|string $user
* @param string $action
* @param string $model
* @param string $title
* @param array $validationErrors
* @param array $fullObject
* @throws Exception
*/
public function validationError($user, $action, $model, $title, array $validationErrors, array $fullObject)
{
$this->log($title, LOG_WARNING);
$change = 'Validation errors: ' . json_encode($validationErrors) . ' Full ' . $model . ': ' . json_encode($fullObject);
$this->createLogEntry($user, $action, $model, 0, $title, $change);
}
// to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version
// this function remedies a fixed upgrade bug instance by eliminating the massive number of erroneous upgrade log entries
public function pruneUpdateLogs($jobId = false, $user)
@ -291,32 +310,31 @@ class Log extends AppModel
));
}
public function pruneUpdateLogsRouter($user)
{
if (Configure::read('MISP.background_jobs')) {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'default',
'job_type' => 'prune_update_logs',
'job_input' => 'All update entries',
'status' => 0,
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => 'Purging the heretic.',
$jobId = $job->createJob(
$user,
Job::WORKER_DEFAULT,
'prune_update_logs',
'All update entries',
'Purging the heretic.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('prune_update_logs', $jobId, $user['id']),
true
return $this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'prune_update_logs',
$jobId,
$user['id']
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
return $process_id;
} else {
$result = $this->pruneUpdateLogs(false, $user);
return $result;

128
app/Model/MispObject.php
View File

@ -1,6 +1,8 @@
<?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
App::uses('AttributeValidationTool', 'Tools');
App::uses('FileAccessTool', 'Tools');
/**
* @property Event $Event
@ -69,7 +71,6 @@ class MispObject extends AppModel
'unique' => array(
'rule' => 'isUnique',
'message' => 'The UUID provided is not unique',
'required' => true,
'on' => 'create'
),
),
@ -216,23 +217,19 @@ class MispObject extends AppModel
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
public function datetimeOrNull($fields)
{
$seen = array_values($fields)[0];
if ($seen === null) {
return true;
}
return strtotime($seen) !== false;
}
public function validateLastSeenValue($fields)
{
$ls = $fields['last_seen'];
if (!isset($this->data['Object']['first_seen']) || is_null($ls)) {
if (!isset($this->data['Object']['first_seen']) || $ls === null) {
return true;
}
$converted = $this->Attribute->ISODatetimeToUTC(['Object' => [
@ -247,50 +244,56 @@ class MispObject extends AppModel
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
foreach ($results as &$v) {
$object = &$v['Object'];
if (!empty($object['first_seen'])) {
$object['first_seen'] = $this->microTimestampToIso($object['first_seen']);
}
if (!empty($object['last_seen'])) {
$object['last_seen'] = $this->microTimestampToIso($object['last_seen']);
}
}
return $results;
}
public function beforeSave($options = array()) {
public function beforeSave($options = array())
{
// generate UUID if it doesn't exist
if (empty($this->data['Object']['uuid'])) {
$this->data['Object']['uuid'] = CakeText::uuid();
}
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
}
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data[$this->alias]['comment'])) {
$this->data[$this->alias]['comment'] = "";
}
// generate UUID if it doesn't exist
if (empty($this->data[$this->alias]['uuid'])) {
$this->data[$this->alias]['uuid'] = CakeText::uuid();
$object = &$this->data['Object'];
if (empty($object['comment'])) {
$object['comment'] = "";
}
// generate timestamp if it doesn't exist
if (empty($this->data[$this->alias]['timestamp'])) {
$date = new DateTime();
$this->data[$this->alias]['timestamp'] = $date->getTimestamp();
if (empty($object['timestamp'])) {
$object['timestamp'] = time();
}
// parse first_seen different formats
if (isset($this->data[$this->alias]['first_seen'])) {
$this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
if (isset($object['first_seen'])) {
$object['first_seen'] = $object['first_seen'] === '' ? null : $object['first_seen'];
}
// parse last_seen different formats
if (isset($this->data[$this->alias]['last_seen'])) {
$this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
if (isset($object['last_seen'])) {
$object['last_seen'] = $object['last_seen'] === '' ? null : $object['last_seen'];
}
if (empty($this->data[$this->alias]['template_version'])) {
$this->data[$this->alias]['template_version'] = 1;
if (empty($object['template_version'])) {
$object['template_version'] = 1;
}
if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) {
$this->data[$this->alias]['deleted'] = 0;
if (isset($object['deleted']) && empty($object['deleted'])) {
$object['deleted'] = 0;
}
if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) {
$this->data['Object']['sharing_group_id'] = 0;
if (!isset($object['distribution']) || $object['distribution'] != 4) {
$object['sharing_group_id'] = 0;
}
if (!isset($this->data[$this->alias]['distribution'])) {
$this->data['Object']['distribution'] = 5;
if (!isset($object['distribution'])) {
$object['distribution'] = 5;
}
return true;
}
@ -359,7 +362,7 @@ class MispObject extends AppModel
}
}
public function checkForDuplicateObjects($object, $eventId, &$duplicatedObjectID)
public function checkForDuplicateObjects($object, $eventId, &$duplicatedObjectID, &$duplicateObjectUuid)
{
$newObjectAttributes = array();
if (isset($object['Object']['Attribute'])) {
@ -373,7 +376,7 @@ class MispObject extends AppModel
$attribute['value'] = $attribute['value'] . '|' . md5(base64_decode($attribute['data']));
}
}
$attributeValueAfterModification = $this->Attribute->modifyBeforeValidation($attribute['type'], $attribute['value']);
$attributeValueAfterModification = AttributeValidationTool::modifyBeforeValidation($attribute['type'], $attribute['value']);
$attributeValueAfterModification = $this->Attribute->runRegexp($attribute['type'], $attributeValueAfterModification);
$newObjectAttributes[] = sha1($attribute['object_relation'] . $attribute['category'] . $attribute['type'] . $attributeValueAfterModification, true);
@ -384,6 +387,7 @@ class MispObject extends AppModel
if ($newObjectAttributeCount === count($previousNewObject)) {
if (empty(array_diff($previousNewObject, $newObjectAttributes))) {
$duplicatedObjectID = $previousNewObject['Object']['id'];
$duplicateObjectUuid = $previousNewObject['Object']['uuid'];
return true;
}
}
@ -442,9 +446,10 @@ class MispObject extends AppModel
$object['Object']['event_id'] = $eventId;
if ($breakOnDuplicate) {
$duplicatedObjectID = null;
$duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID);
$duplicateObjectUuid = null;
$duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID, $dupicateObjectUuid);
if ($duplicate) {
return array('value' => array(__('Duplicate object found (id: %s). Since breakOnDuplicate is set the object will not be added.', $duplicatedObjectID)));
return array('value' => array(__('Duplicate object found (id: %s, uuid: %s). Since breakOnDuplicate is set the object will not be added.', $duplicatedObjectID, $dupicateObjectUuid)));
}
}
$this->create();
@ -729,6 +734,8 @@ class MispObject extends AppModel
* Clean the attribute list up from artifacts introduced by the object form
* @param array $attributes
* @return string|array
* @throws InternalErrorException
* @throws Exception
*/
public function attributeCleanup($attributes)
{
@ -749,23 +756,19 @@ class MispObject extends AppModel
if (isset($attribute['Attachment'])) {
// Check if there were problems with the file upload
// only keep the last part of the filename, this should prevent directory attacks
$filename = basename($attribute['Attachment']['name']);
$tmpfile = new File($attribute['Attachment']['tmp_name']);
if ((isset($attribute['Attachment']['error']) && $attribute['Attachment']['error'] == 0) ||
(!empty($attribute['Attachment']['tmp_name']) && $attribute['Attachment']['tmp_name'] != 'none')
) {
if (!is_uploaded_file($tmpfile->path)) {
if (!is_uploaded_file($attribute['Attachment']['tmp_name'])) {
throw new InternalErrorException('PHP says file was not uploaded. Are you attacking me?');
}
} else {
return 'Issues with the file attachment for the ' . $attribute['object_relation'] . ' attribute. The error code returned is ' . $attribute['Attachment']['error'];
throw new InternalErrorException('Issues with the file attachment for the ' . $attribute['object_relation'] . ' attribute. The error code returned is ' . $attribute['Attachment']['error']);
}
$attributes['Attribute'][$k]['value'] = $attribute['Attachment']['name'];
unset($attributes['Attribute'][$k]['Attachment']);
$attributes['Attribute'][$k]['encrypt'] = $attribute['type'] == 'malware-sample' ? 1 : 0;
$attributes['Attribute'][$k]['data'] = base64_encode($tmpfile->read());
$tmpfile->delete();
$tmpfile->close();
$attributes['Attribute'][$k]['encrypt'] = $attribute['type'] === 'malware-sample' ? 1 : 0;
$attributes['Attribute'][$k]['data'] = base64_encode(FileAccessTool::readAndDelete($attribute['Attachment']['tmp_name']));
}
if (!isset($attributes['Attribute'][$k]['first_seen'])) {
$attributes['Attribute'][$k]['first_seen'] = null;
@ -978,10 +981,11 @@ class MispObject extends AppModel
}
if (!empty($object['Object']['breakOnDuplicate']) || $breakOnDuplicate) {
$duplicatedObjectID = null;
$duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID);
$duplicateObjectUuid = null;
$duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID, $duplicateObjectUuid);
if ($duplicate) {
$this->loadLog()->createLogEntry($user, 'add', 'Object', 0,
__('Object dropped due to it being a duplicate (ID: %s) and breakOnDuplicate being requested for Event %s', $duplicatedObjectID, $eventId),
__('Object dropped due to it being a duplicate (ID: %s, UUID: %s) and breakOnDuplicate being requested for Event %s', $duplicatedObjectID, $dupicateObjectUuid, $eventId),
'Duplicate object found.'
);
return true;
@ -1157,15 +1161,15 @@ class MispObject extends AppModel
*/
public function updateTimestamp($id, $timestamp = false)
{
$object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $id)
));
$object['Object']['timestamp'] = $timestamp === false ? time() : $timestamp;
$object['Object']['skip_zmq'] = 1;
$object['Object']['skip_kafka'] = 1;
$result = $this->save($object);
return $result;
$object = [
'Object' => [
'id' => $id,
'timestamp' => $timestamp === false ? time() : $timestamp,
'skip_zmq' => 1,
'skip_kafka' => 1,
],
];
return $this->save($object, true, ['timestamp']);
}
// Hunt down all LEDA and CASTOR clones

45
app/Model/Module.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('JsonTool', 'Tools');
class Module extends AppModel
{
@ -234,28 +235,27 @@ class Module extends AppModel
* @param array|null $postData
* @param string $moduleFamily
* @return array
* @throws JsonException
* @throws HttpSocketJsonException
* @throws Exception
*/
public function sendRequest($uri, $timeout, $postData = null, $moduleFamily = 'Enrichment')
{
$url = $this->__getModuleServer($moduleFamily);
if (!$url) {
$serverUrl = $this->__getModuleServer($moduleFamily);
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocket', 'Network/Http');
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') {
$settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocket(['timeout' => $timeout]);
$request = array(
'header' => array(
'Content-Type' => 'application/json',
)
);
if ($moduleFamily == 'Cortex') {
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
$request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey');
}
@ -264,26 +264,23 @@ class Module extends AppModel
if (!is_array($postData)) {
throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given.");
}
$post = json_encode($postData);
$response = $httpSocket->post($url . $uri, $post, $request);
$post = JsonTool::encode($postData);
$request['header']['Content-Type'] = 'application/json';
$response = $httpSocket->post($serverUrl . $uri, $post, $request);
} else {
if ($moduleFamily == 'Cortex') {
unset($request['header']['Content-Type']);
}
$response = $httpSocket->get($url . $uri, false, $request);
$response = $httpSocket->get($serverUrl . $uri, false, $request);
}
if (!$response->isOk()) {
if ($httpSocket->lastError()) {
throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']);
}
throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
$e = new HttpSocketHttpException($response, $serverUrl . $uri);
throw new Exception("Failed to get response from `$moduleFamily` module", 0, $e);
}
return $this->jsonDecode($response->body);
return $response->json();
}
/**
* @param string $moduleFamily
* @return array
* @throws JsonException
*/
public function getModuleSettings($moduleFamily = 'Enrichment')
{

69
app/Model/ObjectReference.php
View File

@ -38,17 +38,38 @@ class ObjectReference extends AppModel
)
);
public $validate = [
'uuid' => 'uuid',
'object_id' => [
'rule' => 'numeric',
'required' => true,
'on' => 'create',
],
'event_id' => [
'rule' => 'numeric',
'required' => true,
'on' => 'create',
],
'source_uuid' => 'uuid',
'referenced_uuid' => 'uuid',
'referenced_id' => 'numeric',
'referenced_type' => [
'rule' => ['inList', ['0', '1']],
],
'deleted' => 'boolean',
];
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['ObjectReference']['uuid'])) {
$this->data['ObjectReference']['uuid'] = CakeText::uuid();
$reference = &$this->data['ObjectReference'];
if (empty($reference['uuid'])) {
$reference['uuid'] = CakeText::uuid();
}
if (empty($this->data['ObjectReference']['timestamp'])) {
$this->data['ObjectReference']['timestamp'] = time();
if (empty($reference['timestamp'])) {
$reference['timestamp'] = time();
}
if (!isset($this->data['ObjectReference']['comment'])) {
$this->data['ObjectReference']['comment'] = '';
if (!isset($reference['comment'])) {
$reference['comment'] = '';
}
return true;
}
@ -78,18 +99,24 @@ class ObjectReference extends AppModel
return true;
}
public function updateTimestamps($id, $objectReference = false)
/**
* @param int|array $objectReference
* @return false|void
* @throws Exception
*/
public function updateTimestamps($objectReference)
{
if (!$objectReference) {
if (is_numeric($objectReference)) {
$objectReference = $this->find('first', array(
'recursive' => -1,
'conditions' => array('ObjectReference.id' => $id),
'conditions' => array('ObjectReference.id' => $objectReference),
'fields' => array('event_id', 'object_id')
));
if (empty($objectReference)) {
return false;
}
}
if (empty($objectReference)) {
return false;
}
if (!isset($objectReference['ObjectReference'])) {
$objectReference = array('ObjectReference' => $objectReference);
}
@ -165,12 +192,18 @@ class ObjectReference extends AppModel
if (!$result) {
return $this->validationErrors;
} else {
$this->updateTimestamps($this->id, $objectReference);
$this->updateTimestamps($objectReference);
}
return true;
}
public function captureReference($reference, $eventId, $user)
/**
* @param array $reference
* @param int $eventId
* @return array|bool
* @throws Exception
*/
public function captureReference(array $reference, $eventId)
{
if (isset($reference['uuid'])) {
$existingReference = $this->find('first', array(
@ -257,6 +290,9 @@ class ObjectReference extends AppModel
$reference['object_uuid'] = $sourceObject['Object']['uuid'];
$reference['event_id'] = $eventId;
$result = $this->save(array('ObjectReference' => $reference));
if (!$result) {
return $this->validationErrors;
}
return true;
}
@ -308,7 +344,8 @@ class ObjectReference extends AppModel
return array($referenced_id, $referenced_uuid, $referenced_type);
}
function isValidExtendedEventForReference($sourceEvent, $targetEventID, $user) {
private function isValidExtendedEventForReference(array $sourceEvent, $targetEventID, array $user)
{
if ($sourceEvent['Event']['orgc_id'] != $user['org_id']) {
return false;
}

83
app/Model/ObjectTemplate.php
View File

@ -1,7 +1,11 @@
<?php
App::uses('AppModel', 'Model');
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
/**
* @property ObjectTemplateElement $ObjectTemplateElement
*/
class ObjectTemplate extends AppModel
{
public $actsAs = array(
@ -29,16 +33,14 @@ class ObjectTemplate extends AppModel
'dependent' => true,
)
);
public $validate = array(
);
public $objectsDir = APP . 'files/misp-objects/objects';
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (isset($results[$k]['ObjectTemplate']['requirements'])) {
$results[$k]['ObjectTemplate']['requirements'] = json_decode($results[$k]['ObjectTemplate']['requirements'], true);
if (isset($result['ObjectTemplate']['requirements'])) {
$results[$k]['ObjectTemplate']['requirements'] = json_decode($result['ObjectTemplate']['requirements'], true);
}
}
return $results;
@ -65,9 +67,8 @@ class ObjectTemplate extends AppModel
if (!file_exists($this->objectsDir . DS . $dir . DS . 'definition.json')) {
continue;
}
$file = new File($this->objectsDir . DS . $dir . DS . 'definition.json');
$template = json_decode($file->read(), true);
$file->close();
$file = FileAccessTool::readFromFile($this->objectsDir . DS . $dir . DS . 'definition.json');
$template = JsonTool::decode($file);
if (!isset($template['version'])) {
$template['version'] = 1;
}
@ -98,7 +99,6 @@ class ObjectTemplate extends AppModel
private function __updateObjectTemplate($template, $current, $user = false)
{
$success = false;
$template['requirements'] = array();
$requirementFields = array('required', 'requiredOneOf');
foreach ($requirementFields as $field) {
@ -121,14 +121,16 @@ class ObjectTemplate extends AppModel
}
$id = $this->id;
$this->setActive($id);
$fieldsToCompare = array('object_relation', 'type', 'ui-priority', 'categories', 'sane_default', 'values_list', 'multiple', 'disable_correlation');
$attributes = [];
foreach ($template['attributes'] as $k => $attribute) {
$attribute['object_relation'] = $k;
$attribute = $this->__convertJSONToElement($attribute);
$this->ObjectTemplateElement->create();
$attribute['object_relation'] = $k;
$attribute['object_template_id'] = $id;
$result = $this->ObjectTemplateElement->save(array('ObjectTemplateElement' => $attribute));
$attributes[] = ['ObjectTemplateElement' => $attribute];
}
$this->ObjectTemplateElement->saveMany($attributes);
return true;
}
@ -136,17 +138,17 @@ class ObjectTemplate extends AppModel
{
$result = array();
$translation_table = array(
'misp-usage-frequency' => 'frequency',
'misp-attribute' => 'type',
'description' => 'description',
'ui-priority' => 'ui-priority',
'type' => 'type',
'disable_correlation' => 'disable_correlation',
'object_relation' => 'object_relation',
'categories' => 'categories',
'sane_default' => 'sane_default',
'values_list' => 'values_list',
'multiple' => 'multiple'
'misp-usage-frequency' => 'frequency',
'misp-attribute' => 'type',
'description' => 'description',
'ui-priority' => 'ui-priority',
'type' => 'type',
'disable_correlation' => 'disable_correlation',
'object_relation' => 'object_relation',
'categories' => 'categories',
'sane_default' => 'sane_default',
'values_list' => 'values_list',
'multiple' => 'multiple'
);
foreach ($translation_table as $from => $to) {
if (isset($attribute[$from])) {
@ -291,18 +293,20 @@ class ObjectTemplate extends AppModel
{
$template = $this->find('first', array(
'recursive' => -1,
'conditions' => array('ObjectTemplate.id' => $id)
'conditions' => array('ObjectTemplate.id' => $id),
'fields' => ['ObjectTemplate.id', 'ObjectTemplate.uuid', 'ObjectTemplate.active'],
));
if (empty($template)) {
return false;
}
if ($template['ObjectTemplate']['active']) {
$template['ObjectTemplate']['active'] = 0;
$this->save($template);
$this->save($template, true, ['active']);
return 0;
}
$similar_templates = $this->find('all', array(
'recursive' => -1,
'fields' => ['ObjectTemplate.id'],
'conditions' => array(
'ObjectTemplate.uuid' => $template['ObjectTemplate']['uuid'],
'NOT' => array(
@ -311,42 +315,41 @@ class ObjectTemplate extends AppModel
)
));
$template['ObjectTemplate']['active'] = 1;
$this->save($template);
$this->save($template, true, ['active']);
foreach ($similar_templates as $st) {
$st['ObjectTemplate']['active'] = 0;
$this->save($st);
$this->save($st, true, ['active']);
}
return 1;
}
public function getRawFromDisk($uuidOrName)
{
$template = [];
if (Validation::uuid($uuidOrName)) {
foreach ($this->readTemplatesFromDisk() as $templateFromDisk) {
if ($templateFromDisk['uuid'] == $uuidOrName) {
$template = $templateFromDisk;
break;
if ($templateFromDisk['uuid'] === $uuidOrName) {
return $templateFromDisk;
}
}
} else {
$allTemplateNames = $this->getTemplateDirectoryPaths(false);
if (in_array($uuidOrName, $allTemplateNames)) { // ensure the path is not out of scope
$template = $this->readTemplateFromDisk($this->getFullPathFromTemplateName($uuidOrName));
if (in_array($uuidOrName, $allTemplateNames, true)) { // ensure the path is not out of scope
return $this->readTemplateFromDisk($this->getFullPathFromTemplateName($uuidOrName));
}
}
return $template;
return [];
}
/**
* @throws Exception
*/
private function readTemplateFromDisk($path)
{
$file = new File($path, false);
if (!$file->exists()) {
if (!file_exists($path)) {
return false;
}
$template = json_decode($file->read(), true);
$file->close();
return $template;
$content = FileAccessTool::readFromFile($path);
return JsonTool::decode($content);
}
private function readTemplatesFromDisk()

42
app/Model/Post.php
View File

@ -32,27 +32,33 @@ class Post extends AppModel
{
if (Configure::read('MISP.background_jobs')) {
$user = $this->User->findById($user_id);
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'email',
'job_type' => 'posts_alert',
'job_input' => 'Post: ' . $post_id,
'status' => 0,
'retries' => 0,
'org_id' => $user['User']['org_id'],
'org' => $user['Organisation']['name'],
'message' => 'Sending...',
$jobId = $job->createJob(
$user['User'],
Job::WORKER_EMAIL,
'posts_alert',
'Post: ' . $post_id,
'Sending...'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'email',
'EventShell',
array('postsemail', $user_id, $post_id, $event_id, $title, $message, $jobId),
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::EMAIL_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'postsemail',
$user_id,
$post_id,
$event_id,
$title,
$message,
$jobId
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
return true;
} else {
return $this->sendPostsEmail($user_id, $post_id, $event_id, $title, $message);

1624
app/Model/Server.php
View File

File diff suppressed because it is too large Load Diff

112
app/Model/ShadowAttribute.php
View File

@ -5,6 +5,7 @@ App::uses('File', 'Utility');
App::uses('AttachmentTool', 'Tools');
App::uses('ComplexTypeTool', 'Tools');
App::uses('ServerSyncTool', 'Tools');
App::uses('AttributeValidationTool', 'Tools');
/**
* @property Event $Event
@ -129,13 +130,13 @@ class ShadowAttribute extends AppModel
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
'message' => array('Invalid ISO 8601 format'),
),
'last_seen' => array(
'datetimeOrNull' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
'message' => array('Invalid ISO 8601 format'),
),
'validateLastSeenValue' => array(
'rule' => array('validateLastSeenValue'),
@ -173,7 +174,7 @@ class ShadowAttribute extends AppModel
$compositeTypes = $this->getCompositeTypes();
// explode composite types in value1 and value2
$pieces = explode('|', $this->data['ShadowAttribute']['value']);
if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes)) {
if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes, true)) {
if (2 != count($pieces)) {
throw new InternalErrorException('Composite type, but value not explodable');
}
@ -300,58 +301,67 @@ class ShadowAttribute extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
// remove leading and trailing blanks
//$this->trimStringFields(); // TODO
if (!isset($this->data['ShadowAttribute']['comment'])) {
$this->data['ShadowAttribute']['comment'] = '';
}
if (!isset($this->data['ShadowAttribute']['type'])) {
$proposal = &$this->data['ShadowAttribute'];
if (!isset($proposal['type'])) {
$this->invalidate('type', 'No value provided.');
return false;
}
if (!isset($proposal['comment'])) {
$proposal['comment'] = '';
}
// make some changes to the inserted value
if (isset($this->data['ShadowAttribute']['value'])) {
$value = trim($this->data['ShadowAttribute']['value']);
$value = ComplexTypeTool::refangValue($value, $this->data['ShadowAttribute']['type']);
$value = $this->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $value);
$this->data['ShadowAttribute']['value'] = $value;
if (isset($proposal['value'])) {
$value = trim($proposal['value']);
$value = ComplexTypeTool::refangValue($value, $proposal['type']);
$value = AttributeValidationTool::modifyBeforeValidation($proposal['type'], $value);
$proposal['value'] = $value;
}
if (!isset($this->data['ShadowAttribute']['org'])) {
$this->data['ShadowAttribute']['org'] = '';
if (!isset($proposal['org'])) {
$proposal['org'] = '';
}
if (empty($this->data['ShadowAttribute']['timestamp'])) {
$date = new DateTime();
$this->data['ShadowAttribute']['timestamp'] = $date->getTimestamp();
if (empty($proposal['timestamp'])) {
$proposal['timestamp'] = time();
}
if (!isset($this->data['ShadowAttribute']['proposal_to_delete'])) {
$this->data['ShadowAttribute']['proposal_to_delete'] = 0;
if (!isset($proposal['proposal_to_delete'])) {
$proposal['proposal_to_delete'] = 0;
}
// generate UUID if it doesn't exist
if (empty($this->data['ShadowAttribute']['uuid'])) {
$this->data['ShadowAttribute']['uuid'] = CakeText::uuid();
if (empty($proposal['uuid'])) {
$proposal['uuid'] = CakeText::uuid();
} else {
$this->data['ShadowAttribute']['uuid'] = strtolower($this->data['ShadowAttribute']['uuid']);
$proposal['uuid'] = strtolower($proposal['uuid']);
}
if (!empty($this->data['ShadowAttribute']['type']) && empty($this->data['ShadowAttribute']['category'])) {
$this->data['ShadowAttribute']['category'] = $this->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
if (empty($proposal['category'])) {
$proposal['category'] = $this->Attribute->typeDefinitions[$proposal['type']]['default_category'];
}
if (isset($proposal['first_seen'])) {
$proposal['first_seen'] = $proposal['first_seen'] === '' ? null : $proposal['first_seen'];
}
if (isset($proposal['last_seen'])) {
$proposal['last_seen'] = $proposal['last_seen'] === '' ? null : $proposal['last_seen'];
}
// always return true, otherwise the object cannot be saved
return true;
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
foreach ($results as &$v) {
$proposal = &$v['ShadowAttribute'];
if (!empty($proposal['first_seen'])) {
$proposal['first_seen'] = $this->microTimestampToIso($proposal['first_seen']);
}
if (!empty($proposal['last_seen'])) {
$proposal['last_seen'] = $this->microTimestampToIso($proposal['last_seen']);
}
}
return $results;
}
@ -373,7 +383,7 @@ class ShadowAttribute extends AppModel
public function validateAttributeValue($fields)
{
$value = $fields['value'];
return $this->Attribute->runValidation($value, $this->data['ShadowAttribute']['type']);
return AttributeValidationTool::validate($this->data['ShadowAttribute']['type'], $value);
}
public function getCompositeTypes()
@ -803,26 +813,28 @@ class ShadowAttribute extends AppModel
));
}
} else {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'default',
'job_type' => 'generate proposal correlation',
'job_input' => 'All attributes',
'retries' => 0,
'status' => 1,
'org' => 'SYSTEM',
'message' => 'Correlating Proposals.',
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
'generate proposal correlation',
'All attributes',
'Correlating Proposals.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('jobGenerateShadowAttributeCorrelation', $jobId),
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'jobGenerateShadowAttributeCorrelation',
$jobId
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',

9
app/Model/SharingGroup.php
View File

@ -5,6 +5,9 @@ App::uses('AppModel', 'Model');
* @property SharingGroupOrg $SharingGroupOrg
* @property SharingGroupServer $SharingGroupServer
* @property Organisation $Organisation
* @property Event $Event
* @property Attribute $Attribute
* @property Thread $Thread
*/
class SharingGroup extends AppModel
{
@ -101,6 +104,12 @@ class SharingGroup extends AppModel
if ($this->Attribute->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
if ($this->Attribute->Object->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
if ($this->Event->EventReport->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
return true;
}

47
app/Model/SharingGroupOrg.php
View File

@ -1,29 +1,41 @@
<?php
App::uses('AppModel', 'Model');
class SharingGroupOrg extends AppModel
{
public $actsAs = array('AuditLog', 'Containable');
public $belongsTo = array(
'SharingGroup' => array(
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
),
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id',
//'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid')
)
'SharingGroup' => array(
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
),
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id',
//'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid')
)
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
$data = $this->data[$this->alias];
$conditions = [
'sharing_group_id' => $data['sharing_group_id'],
'org_id' => $data['org_id'],
];
if (isset($data['id'])) {
$conditions['id !='] = $data['id'];
}
if ($this->hasAny($conditions)) {
$this->log("Trying to save duplicate organisation `{$data['org_id']}` for sharing group `{$data['sharing_group_id']}. This should never happened.");
$this->invalidate('org_id', 'The same organisation is already assigned to this sharing group.');
return false;
}
}
public function updateOrgsForSG($id, $new_orgs, $old_orgs, $user)
{
$log = ClassRegistry::init('Log');
// Loop through all of the organisations we want to add.
foreach ($new_orgs as $org) {
$SgO = array(
@ -54,16 +66,16 @@ class SharingGroupOrg extends AppModel
}
if ($this->save($SgO)) {
if ($isChange) {
$log->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.'));
$this->loadLog()->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.'));
} else {
$log->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : ''));
$this->loadLog()->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : ''));
}
}
}
// We are left with some "old orgs" that are not in the new list. This means that they can be safely deleted.
foreach ($old_orgs as $old_org) {
if ($this->delete($old_org['id'])) {
$log->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.');
$this->loadLog()->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.');
}
}
}
@ -83,7 +95,12 @@ class SharingGroupOrg extends AppModel
return $sgs;
}
// pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object
/**
* Pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object
* @param int $id
* @param int $org_id
* @return bool
*/
public function checkIfAuthorised($id, $org_id)
{
return $this->hasAny([

10
app/Model/Sighting.php
View File

@ -49,7 +49,7 @@ class Sighting extends AppModel
),
);
public $type = array(
const TYPE = array(
0 => 'sighting',
1 => 'false-positive',
2 => 'expiration'
@ -63,7 +63,6 @@ class Sighting extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['Sighting']['id']) && empty($this->data['Sighting']['date_sighting'])) {
$this->data['Sighting']['date_sighting'] = date('Y-m-d H:i:s');
}
@ -77,7 +76,7 @@ class Sighting extends AppModel
public function afterSave($created, $options = array())
{
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable');
$pubToZmq = $this->pubToZmq('sighting');
$kafkaTopic = $this->kafkaTopic('sighting');
if ($pubToZmq || $kafkaTopic) {
$user = array(
@ -101,8 +100,7 @@ class Sighting extends AppModel
public function beforeDelete($cascade = true)
{
parent::beforeDelete();
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable');
$pubToZmq = $this->pubToZmq('sighting');
$kafkaTopic = $this->kafkaTopic('sighting');
if ($pubToZmq || $kafkaTopic) {
$user = array(
@ -441,7 +439,7 @@ class Sighting extends AppModel
$sparklineData = [];
$range = $this->getMaximumRange();
foreach ($groupedSightings as $sighting) {
$type = $this->type[$sighting['type']];
$type = self::TYPE[$sighting['type']];
$orgName = isset($sighting['Organisation']['name']) ? $sighting['Organisation']['name'] : __('Others');
$count = (int)$sighting['sighting_count'];
$inRange = strtotime($sighting['date']) >= $range;

170
app/Model/SystemSetting.php Normal file
View File

@ -0,0 +1,170 @@
<?php
App::uses('AppModel', 'Model');
App::uses('EncryptedValue', 'Tools');
App::uses('BetterSecurity', 'Tools');
class SystemSetting extends AppModel
{
public $actsAs = [
'AuditLog'
];
public $primaryKey = 'setting';
// Blocked setting that cannot be saved or fetched from DB. The same as cli_only settings.
const BLOCKED_SETTINGS = [
'Security.encryption_key',
'Security.disable_local_feed_access',
'GnuPG.binary',
'MISP.python_bin',
'MISP.ca_path',
'MISP.tmpdir',
'MISP.system_setting_db',
'MISP.attachments_dir',
];
// Allow to set config values just for these categories
const ALLOWED_CATEGORIES = [
'MISP',
'Security',
'GnuPG',
'SMIME',
'Proxy',
'SecureAuth',
'Session',
'Plugin',
'debug',
'site_admin_debug',
];
/**
* Set config values from database into global Configure class
*/
public static function setGlobalSetting()
{
/** @var self $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
if (!$systemSetting->databaseExists()) {
return;
}
$settings = $systemSetting->getSettings();
foreach ($settings as $settingName => $settingValue) {
$firstPart = explode('.', $settingName)[0];
if (in_array($firstPart, self::ALLOWED_CATEGORIES, true) && !in_array($settingName, self::BLOCKED_SETTINGS, true)) {
Configure::write($settingName, $settingValue);
}
}
}
public function databaseExists()
{
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
return in_array('system_settings', $tables, true);
}
/**
* @return array
* @throws JsonException
*/
public function getSettings()
{
$settings = $this->find('list', [
'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
]);
return array_map(function ($value) {
if (EncryptedValue::isEncrypted($value)) {
return new EncryptedValue($value, true);
} else {
return JsonTool::decode($value);
}
}, $settings);
}
/**
* @param string $setting Setting name
* @param mixed $value
* @throws Exception
*/
public function setSetting($setting, $value)
{
$firstPart = explode('.', $setting)[0];
if (!in_array($firstPart, self::ALLOWED_CATEGORIES, true) || in_array($setting, self::BLOCKED_SETTINGS, true)) {
return false; // blocked setting
}
if ($value === '' || $value === null) {
if ($this->hasAny(['SystemSetting.setting' => $setting])) {
return $this->delete($setting); // delete the whole setting when value is empty
}
return true;
}
$value = JsonTool::encode($value);
// If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it
if (self::isSensitive($setting)) {
$value = EncryptedValue::encryptIfEnabled($value);
}
$valid = $this->save(['SystemSetting' => [
'setting' => $setting,
'value' => $value,
]]);
if (!$valid) {
throw new Exception("Could not save system setting `$setting` because of validation errors: " . JsonTool::encode($this->validationErrors));
}
return true;
}
/**
* @param string|null $old Old (or current) encryption key.
* @param string|null $new New encryption key. If empty, encrypted values will be decrypted.
* @throws Exception
*/
public function reencrypt($old, $new)
{
$settings = $this->find('list', [
'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
]);
$toSave = [];
foreach ($settings as $setting => $value) {
if (!self::isSensitive($setting)) {
continue;
}
if (EncryptedValue::isEncrypted($value)) {
try {
$value = BetterSecurity::decrypt(substr($value, 2), $old);
} catch (Exception $e) {
throw new Exception("Could not decrypt `$setting` setting.", 0, $e);
}
}
if (!empty($new)) {
$value = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $new);
}
$toSave[] = ['SystemSetting' => [
'setting' => $setting,
'value' => $value,
]];
}
if (empty($toSave)) {
return true;
}
return $this->saveMany($toSave);
}
/**
* Sensitive setting are passwords or api keys.
* @param string $setting Setting name
* @return bool
*/
public static function isSensitive($setting)
{
if ($setting === 'Security.encryption_key' || $setting === 'Security.salt') {
return true;
}
if (substr($setting, 0, 7) === 'Plugin.' && (strpos($setting, 'apikey') !== false || strpos($setting, 'secret') !== false)) {
return true;
}
return strpos($setting, 'password') !== false;
}
}

302
app/Model/Tag.php
View File

@ -4,6 +4,8 @@ App::uses('AppModel', 'Model');
/**
* @property EventTag $EventTag
* @property AttributeTag $AttributeTag
* @property FavouriteTag $FavouriteTag
* @property Organisation $Organisation
*/
class Tag extends AppModel
{
@ -22,28 +24,28 @@ class Tag extends AppModel
);
public $validate = array(
'name' => array(
'required' => array(
'rule' => array('notBlank', 'name'),
'message' => 'This field is required.'
),
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'name'),
),
'unique' => array(
'rule' => 'isUnique',
'message' => 'A similar name already exists.',
),
'name' => array(
'required' => array(
'rule' => array('notBlank', 'name'),
'message' => 'This field is required.'
),
'colour' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'colour'),
),
'userdefined' => array(
'rule' => 'validateColour',
'message' => 'Colour has to be in the RGB format (#FFFFFF)',
),
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'name'),
),
'unique' => array(
'rule' => 'isUnique',
'message' => 'A similar name already exists.',
),
),
'colour' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'colour'),
),
'userdefined' => array(
'rule' => 'validateColour',
'message' => 'Colour has to be in the RGB format (#FFFFFF)',
),
),
);
public $hasMany = array(
@ -83,24 +85,27 @@ class Tag extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (!isset($this->data['Tag']['org_id'])) {
$this->data['Tag']['org_id'] = 0;
$tag = &$this->data['Tag'];
if (!isset($tag['org_id'])) {
$tag['org_id'] = 0;
}
if (!isset($this->data['Tag']['user_id'])) {
$this->data['Tag']['user_id'] = 0;
if (!isset($tag['user_id'])) {
$tag['user_id'] = 0;
}
if (!isset($this->data['Tag']['hide_tag'])) {
$this->data['Tag']['hide_tag'] = Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0;
if (!isset($tag['hide_tag'])) {
$tag['hide_tag'] = Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0;
}
if (!isset($this->data['Tag']['exportable'])) {
$this->data['Tag']['exportable'] = 1;
if (!isset($tag['exportable'])) {
$tag['exportable'] = 1;
}
if (isset($this->data['Tag']['name']) && strlen($this->data['Tag']['name']) >= 255) {
$this->data['Tag']['name'] = substr($this->data['Tag']['name'], 0, 255);
if (!isset($tag['local_only'])) {
$tag['local_only'] = 0;
}
$this->data['Tag']['is_galaxy'] = preg_match($this->reGalaxy, $this->data['Tag']['name']);
$this->data['Tag']['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $this->data['Tag']['name']);
if (isset($tag['name']) && strlen($tag['name']) >= 255) {
$tag['name'] = substr($tag['name'], 0, 255);
}
$tag['is_galaxy'] = preg_match($this->reGalaxy, $tag['name']);
$tag['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $tag['name']);
return true;
}
@ -149,8 +154,7 @@ class Tag extends AppModel
public function afterFind($results, $primary = false)
{
$results = $this->checkForOverride($results);
return $results;
return $this->checkForOverride($results);
}
public function validateColour($fields)
@ -161,12 +165,41 @@ class Tag extends AppModel
return true;
}
/**
* @param array $user
* @param string $tagName
* @return mixed|null
*/
public function lookupTagIdForUser(array $user, $tagName)
{
$conditions = ['LOWER(Tag.name)' => mb_strtolower($tagName)];
if (!$user['Role']['perm_site_admin']) {
$conditions['Tag.org_id'] = [0, $user['org_id']];
$conditions['Tag.user_id'] = [0, $user['id']];
}
$tagId = $this->find('first', array(
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('Tag.id'),
'callbacks' => false,
));
if (empty($tagId)) {
return null;
}
return $tagId['Tag']['id'];
}
/**
* @param string $tagName
* @return int|mixed
*/
public function lookupTagIdFromName($tagName)
{
$tagId = $this->find('first', array(
'conditions' => array('LOWER(Tag.name)' => strtolower($tagName)),
'conditions' => array('LOWER(Tag.name)' => mb_strtolower($tagName)),
'recursive' => -1,
'fields' => array('Tag.id')
'fields' => array('Tag.id'),
'callbacks' => false,
));
if (empty($tagId)) {
return -1;
@ -186,44 +219,14 @@ class Tag extends AppModel
return $this->find('all', array('conditions' => $conditions, 'recursive' => -1));
}
// find all of the tag ids that belong to the accepted tag names and the rejected tag names
public function fetchTagIdsFromFilter($accept = array(), $reject = array())
/**
* @param array $accept
* @param array $reject
* @deprecated Use EventTag::fetchEventTagIds instead
*/
public function fetchEventTagIds($accept, $reject)
{
$results = array(0 => array(), 1 => array());
if (!empty($accept)) {
foreach ($accept as $tag) {
$temp = $this->lookupTagIdFromName($tag);
if (!in_array($temp, $results[0])) {
$results[0][] = $temp;
}
}
}
if (!empty($reject)) {
foreach ($reject as $tag) {
$temp = $this->lookupTagIdFromName($tag);
if (!in_array($temp, $results[1])) {
$results[1][] = $temp;
}
}
}
return $results;
}
// find all of the event Ids that belong to the accepted tags and the rejected tags
public function fetchEventTagIds($accept = array(), $reject = array())
{
$acceptIds = array();
$rejectIds = array();
if (!empty($accept)) {
$acceptIds = $this->findEventIdsByTagNames($accept);
if (empty($acceptIds)) {
$acceptIds[] = -1;
}
}
if (!empty($reject)) {
$rejectIds = $this->findEventIdsByTagNames($reject);
}
return array($acceptIds, $rejectIds);
$this->EventTag->fetchEventTagIds($accept, $reject);
}
// find all of the tag Ids that belong to the accepted tags and the rejected tags
@ -256,78 +259,38 @@ class Tag extends AppModel
return array($acceptIds, $rejectIds);
}
// pass a list of tag names to receive a list of matched tag IDs
/**
* pass a list of tag names to receive a list of matched tag IDs
* @param string|array $array
* @return array|int|null
*/
public function findTagIdsByTagNames($array)
{
$ids = array();
$tag_ids = array();
if (!is_array($array)) {
$array = array($array);
}
foreach ($array as $k => $tag) {
$tagIds = [];
$tagNames = [];
foreach ($array as $tag) {
if (is_numeric($tag)) {
$tag_ids[] = $tag;
unset($array[$k]);
}
}
$array = array_values($array);
if (!empty($array)) {
foreach ($array as $a) {
$conditions['OR'][] = array('Tag.name like' => $a);
}
$params = array(
'recursive' => 1,
'conditions' => $conditions,
'fields' => array('Tag.id', 'Tag.id')
);
$result = $this->find('list', $params);
$tag_ids = array_merge($result, $tag_ids);
}
return array_values($tag_ids);
}
public function findEventIdsByTagNames($array)
{
$ids = array();
foreach ($array as $a) {
if (is_numeric($a)) {
$conditions['OR'][] = array('id' => $a);
$tagIds[] = $tag;
} else {
$conditions['OR'][] = array('LOWER(name) like' => strtolower($a));
$tagNames[] = $tag;
}
}
$params = array(
if (!empty($tagNames)) {
$conditions = [];
foreach ($tagNames as $tagName) {
$conditions[] = array('Tag.name LIKE' => $tagName);
}
$result = $this->find('column', array(
'recursive' => 1,
'contain' => 'EventTag',
'conditions' => $conditions
);
$result = $this->find('all', $params);
foreach ($result as $tag) {
foreach ($tag['EventTag'] as $eventTag) {
$ids[] = $eventTag['event_id'];
}
'conditions' => ['OR' => $conditions],
'fields' => array('Tag.id')
));
$tagIds = array_merge($result, $tagIds);
}
return $ids;
}
public function findAttributeIdsByAttributeTagNames($array)
{
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(name) LIKE' => strtolower($a));
}
$params = array(
'recursive' => 1,
'contain' => 'AttributeTag',
'conditions' => $conditions
);
$result = $this->find('all', $params);
foreach ($result as $tag) {
foreach ($tag['AttributeTag'] as $attributeTag) {
$ids[] = $attributeTag['attribute_id'];
}
}
return $ids;
return $tagIds;
}
/**
@ -337,23 +300,25 @@ class Tag extends AppModel
* @return false|int
* @throws Exception
*/
public function captureTag($tag, $user, $force=false)
public function captureTag(array $tag, array $user, $force=false)
{
$existingTag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('LOWER(name)' => mb_strtolower($tag['name'])),
'fields' => ['id', 'org_id', 'user_id'],
'callbacks' => false,
));
if (empty($existingTag)) {
if ($force || $user['Role']['perm_tag_editor']) {
$this->create();
if (!isset($tag['colour']) || empty($tag['colour'])) {
if (empty($tag['colour'])) {
$tag['colour'] = $this->random_color();
}
$tag = array(
'name' => $tag['name'],
'colour' => $tag['colour'],
'exportable' => isset($tag['exportable']) ? $tag['exportable'] : 1,
'local_only' => $tag['local_only'] ?? 0,
'org_id' => 0,
'user_id' => 0,
'hide_tag' => Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0
@ -363,22 +328,21 @@ class Tag extends AppModel
} else {
return false;
}
} else {
if (
!$user['Role']['perm_site_admin'] &&
}
if (
!$user['Role']['perm_site_admin'] &&
(
(
(
$existingTag['Tag']['org_id'] != 0 &&
$existingTag['Tag']['org_id'] != $user['org_id']
) ||
(
$existingTag['Tag']['user_id'] != 0 &&
$existingTag['Tag']['user_id'] != $user['id']
)
$existingTag['Tag']['org_id'] != 0 &&
$existingTag['Tag']['org_id'] != $user['org_id']
) ||
(
$existingTag['Tag']['user_id'] != 0 &&
$existingTag['Tag']['user_id'] != $user['id']
)
) {
return false;
}
)
) {
return false;
}
return $existingTag['Tag']['id'];
}
@ -392,6 +356,13 @@ class Tag extends AppModel
return $colour;
}
/**
* @param string $name
* @param string|false $colour
* @param null $numerical_value
* @return int|false Created tag ID or false on error
* @throws Exception
*/
public function quickAdd($name, $colour = false, $numerical_value = null)
{
$this->create();
@ -401,19 +372,26 @@ class Tag extends AppModel
$data = array(
'name' => $name,
'colour' => $colour,
'exportable' => 1
'exportable' => 1,
);
if (!is_null($numerical_value)) {
if ($numerical_value !== null) {
$data['numerical_value'] = $numerical_value;
}
return ($this->save($data));
if ($this->save(['Tag' => $data])) {
return $this->id;
} else {
return false;
}
}
public function quickEdit($tag, $name, $colour, $hide = false, $numerical_value = null)
public function quickEdit($tag, $name, $colour, $hide = false, $numerical_value = null, $local_only = -1)
{
if ($tag['Tag']['colour'] !== $colour || $tag['Tag']['name'] !== $name || $hide !== false || $tag['Tag']['numerical_value'] !== $numerical_value) {
if ($tag['Tag']['colour'] !== $colour || $tag['Tag']['name'] !== $name || $hide !== false || $tag['Tag']['numerical_value'] !== $numerical_value || ($tag['Tag']['local_only'] !== $local_only && $local_only !== -1)) {
$tag['Tag']['name'] = $name;
$tag['Tag']['colour'] = $colour;
if ($tag['Tag']['local_only'] !== -1) {
$tag['Tag']['local_only'] = $local_only;
}
if ($hide !== false) {
$tag['Tag']['hide_tag'] = $hide;
}
@ -434,15 +412,21 @@ class Tag extends AppModel
}
/**
* Recover user_id from the session and override numerical_values from userSetting
*/
public function checkForOverride($tags)
* Recover user_id from the session and override numerical_values from userSetting.
*
* @param array $tags
* @return array
*/
private function checkForOverride($tags)
{
$userId = Configure::read('CurrentUserId');
if ($this->tagOverrides === false && $userId > 0) {
$this->UserSetting = ClassRegistry::init('UserSetting');
$this->tagOverrides = $this->UserSetting->getTagNumericalValueOverride($userId);
}
if (empty($this->tagOverrides)) {
return $tags;
}
foreach ($tags as $k => $tag) {
if (isset($tag['Tag']['name'])) {
$tagName = $tag['Tag']['name'];

12
app/Model/Taxonomy.php
View File

@ -242,7 +242,7 @@ class Taxonomy extends AppModel
// returns all tags associated to a taxonomy
// returns all tags not associated to a taxonomy if $inverse is true
public function getAllTaxonomyTags($inverse = false, $user = false, $full = false, $hideUnselectable = true)
public function getAllTaxonomyTags($inverse = false, $user = false, $full = false, $hideUnselectable = true, $local_tag = false)
{
$this->Tag = ClassRegistry::init('Tag');
$taxonomyIdList = $this->find('column', array('fields' => array('Taxonomy.id')));
@ -260,6 +260,10 @@ class Taxonomy extends AppModel
if (Configure::read('MISP.incoming_tags_disabled_by_default') || $hideUnselectable) {
$conditions['Tag.hide_tag'] = 0;
}
// If the tag is to be added as global, we filter out the local_only tags
if (!$local_tag) {
$conditions['Tag.local_only'] = 0;
}
if ($full) {
$allTags = $this->Tag->find(
'all',
@ -666,11 +670,15 @@ class Taxonomy extends AppModel
return true;
}
/**
* @param array $tagList
* @return array[]
*/
public function checkIfTagInconsistencies($tagList)
{
$eventTags = array();
$localEventTags = array();
foreach($tagList as $tag) {
foreach ($tagList as $tag) {
if ($tag['local'] == 0) {
$eventTags[] = $tag['Tag']['name'];
} else {

5
app/Model/Template.php
View File

@ -72,9 +72,4 @@ class Template extends AppModel
return false;
}
}
public function generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}
}

108
app/Model/User.php
View File

@ -231,9 +231,9 @@ class User extends AppModel
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$db_version = $this->AdminSetting->getSetting('db_version');
if ($db_version >= 62) {
// bind AuthKey just when authkey table already exists. This is important for updating from old versions
if (in_array('auth_keys', $this->getDataSource()->listSources(), true)) {
$this->bindModel([
'hasMany' => ['AuthKey']
], false);
@ -1067,27 +1067,29 @@ class User extends AppModel
public function resetAllSyncAuthKeysRouter($user, $jobId = false)
{
if (Configure::read('MISP.background_jobs')) {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'prio',
'job_type' => __('reset_all_sync_api_keys'),
'job_input' => __('Reseting all API keys'),
'status' => 0,
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => 'Issuing new API keys to all sync users.',
$jobId = $job->createJob(
$user,
Job::WORKER_PRIO,
'reset_all_sync_api_keys',
__('Reseting all API keys'),
'Issuing new API keys to all sync users.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'AdminShell',
array('resetSyncAuthkeys', $user['id'], $jobId),
true
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'resetSyncAuthkeys',
$user['id'],
$jobId
],
true,
$jobId
);
$job->saveField('process_id', $process_id);
return true;
} else {
return $this->resetAllSyncAuthKeys($user);
@ -1504,4 +1506,68 @@ class User extends AppModel
$banStatus['message'] = __('User email notification ban setting is not enabled');
return $banStatus;
}
/**
* @param array $user
* @return bool
*/
public function hasNotifications(array $user)
{
$hasProposal = $this->Event->ShadowAttribute->hasAny([
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
]);
if ($hasProposal) {
return true;
}
if (Configure::read('MISP.delegation') && $this->_getDelegationCount($user)) {
return true;
}
return false;
}
/**
* @param array $user
* @return array
*/
public function populateNotifications(array $user)
{
$notifications = array();
list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user);
$notifications['total'] = $notifications['proposalCount'];
if (Configure::read('MISP.delegation')) {
$notifications['delegationCount'] = $this->_getDelegationCount($user);
$notifications['total'] += $notifications['delegationCount'];
}
return $notifications;
}
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount($user, $mode = 'full')
{
$results[0] = $this->Event->ShadowAttribute->find('count', [
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
)
]);
$results[1] = $this->Event->ShadowAttribute->find('count', [
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
),
'fields' => 'distinct event_id'
]);
return $results;
}
private function _getDelegationCount($user)
{
$this->EventDelegation = ClassRegistry::init('EventDelegation');
return $this->EventDelegation->find('count', array(
'recursive' => -1,
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
}
}

105
app/Model/UserSetting.php
View File

@ -1,5 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property User $User
*/
class UserSetting extends AppModel
{
public $useTable = 'user_settings';
@ -16,18 +20,15 @@ class UserSetting extends AppModel
);
public $validate = array(
'json' => array(
'isValidJson' => array(
'rule' => array('isValidJson'),
)
)
'value' => 'valueIsJson',
);
public $belongsTo = array(
'User'
);
public $validSettings = array(
// private
const VALID_SETTINGS = array(
'publish_alert_filter' => array(
'placeholder' => array(
'AND' => array(
@ -96,12 +97,14 @@ class UserSetting extends AppModel
'event_index_hide_columns' => [
'placeholder' => ['clusters'],
],
'oidc' => [ // Data saved by OIDC plugin
'restricted' => 'perm_site_admin',
],
);
// massage the data before we send it off for validation before saving anything
public function beforeValidate($options = array())
{
parent::beforeValidate();
// add a timestamp if it is not set
if (empty($this->data['UserSetting']['timestamp'])) {
$this->data['UserSetting']['timestamp'] = time();
@ -124,35 +127,39 @@ class UserSetting extends AppModel
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true);
if (isset($v['UserSetting']['value'])) {
$results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true);
}
}
return $results;
}
public function checkSettingValidity($setting)
{
return isset($this->validSettings[$setting]);
return isset(self::VALID_SETTINGS[$setting]);
}
public function checkSettingAccess($user, $setting)
/**
* @param array $user
* @param string $setting
* @return bool|string
*/
public function checkSettingAccess(array $user, $setting)
{
if (!empty($this->validSettings[$setting]['restricted'])) {
$role_check = $this->validSettings[$setting]['restricted'];
if (!is_array($role_check)) {
$role_check = array($role_check);
if (!empty(self::VALID_SETTINGS[$setting]['restricted'])) {
$roleCheck = self::VALID_SETTINGS[$setting]['restricted'];
if (!is_array($roleCheck)) {
$roleCheck = array($roleCheck);
}
$userHasValidRole = false;
foreach ($role_check as $role) {
foreach ($roleCheck as $role) {
if (!empty($user['Role'][$role])) {
return true;
}
}
if (!$userHasValidRole) {
foreach ($role_check as &$role) {
$role = substr($role, 5);
}
return implode(', ', $role_check);
foreach ($roleCheck as &$role) {
$role = substr($role, 5);
}
return implode(', ', $roleCheck);
}
return true;
}
@ -203,7 +210,7 @@ class UserSetting extends AppModel
return false;
}
public function getDefaulRestSearchParameters($user)
public function getDefaultRestSearchParameters($user)
{
return $this->getValueForUser($user['id'], 'default_restsearch_parameters') ?: [];
}
@ -236,8 +243,8 @@ class UserSetting extends AppModel
/**
* Check whether the event is something the user is interested (to be alerted on)
* @param $user
* @param $event
* @param array $user
* @param array $event
* @return bool
*/
public function checkPublishFilter(array $user, array $event)
@ -295,7 +302,7 @@ class UserSetting extends AppModel
}
}
/*
/**
* Checks if an event matches the given rule
* valid filters:
* - AttributeTag.name
@ -307,6 +314,11 @@ class UserSetting extends AppModel
* Values passed can be used for direct string comparisons or alternatively
* as substring matches by encapsulating the string in a pair of "%" characters
* Each rule can take a list of values
*
* @param string $rule
* @param array|string $lookup_values
* @param array $event
* @return bool
*/
private function __checkEvent($rule, $lookup_values, $event)
{
@ -341,7 +353,7 @@ class UserSetting extends AppModel
$extracted_value = mb_strtolower($extracted_value);
foreach ($lookup_values as $lookup_value) {
$lookup_value_trimmed = trim($lookup_value, "%");
if (strlen($lookup_value_trimmed) != strlen($lookup_value)) {
if (strlen($lookup_value_trimmed) !== strlen($lookup_value)) {
if (strpos($extracted_value, $lookup_value_trimmed) !== false) {
return true;
}
@ -356,7 +368,13 @@ class UserSetting extends AppModel
return false;
}
public function setSetting($user, &$data)
/**
* @param array $user
* @param array $data
* @return bool
* @throws Exception
*/
public function setSetting(array $user, array $data)
{
$userSetting = array();
if (!empty($data['UserSetting']['user_id']) && is_numeric($data['UserSetting']['user_id'])) {
@ -391,21 +409,42 @@ class UserSetting extends AppModel
} else {
$userSetting['value'] = '';
}
return $this->setSettingInternal($userSetting['user_id'], $userSetting['setting'], $userSetting['value']);
}
/**
* Set user setting without checking permission.
* @param int $userId
* @param string $setting
* @param mixed $value
* @return array|bool|mixed|null
* @throws Exception
*/
public function setSettingInternal($userId, $setting, $value)
{
$userSetting = [
'user_id' => $userId,
'setting' => $setting,
'value' => $value,
];
$existingSetting = $this->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.user_id' => $userSetting['user_id'],
'UserSetting.setting' => $userSetting['setting']
)
'UserSetting.user_id' => $userId,
'UserSetting.setting' => $setting,
),
'fields' => ['UserSetting.id'],
'callbacks' => false,
));
if (empty($existingSetting)) {
$this->create();
} else {
$userSetting['id'] = $existingSetting['UserSetting']['id'];
}
// save the setting
$result = $this->save(array('UserSetting' => $userSetting));
return true;
return $this->save($userSetting);
}
/**

Some files were not shown because too many files have changed in this diff Show More