Merge branch '2.4' of github.com:MISP/MISP into feature-report-from-event

pull/6590/head
mokaddem 2020-11-17 10:31:39 +01:00
commit 86b0f75700
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
12 changed files with 342 additions and 74 deletions

225
.github/workflows/main.yml vendored Normal file
View File

@ -0,0 +1,225 @@
# This is a basic workflow to help you get started with Actions
name: misp
# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the 2.4 branch
on:
push:
branches: [ 2.4 ]
pull_request:
branches: [ 2.4 ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
# Stop mysql
- name: Shutdown Ubuntu MySQL (SUDO)
run: sudo service mysql stop
# Run mariadb
- uses: getong/mariadb-action@v1.1
with:
host port: 3306
container port: 3306
mysql root password: 'bar'
mysql database: 'misp'
mysql user: 'misp'
mysql password: 'blah'
- uses: zhulik/redis-action@1.1.0
with:
redis version: '5'
number of databases: 100
# prepare php stuff
- name: Setup PHP
uses: shivammathur/setup-php@v2
with:
php-version: '7.4'
tools: pecl, composer
extensions: mysql, mbstring, json, xml, opcache, readline, redis, gnupg, gd
# Runs a set of commands using the runners shell
- name: Install deps
run: |
git submodule update --init --recursive
date
sudo apt-get -y update
sudo apt-get -y install python3 python3-venv virtualenv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php7.4-dev php7.4-cli libapache2-mod-php libfuzzy-dev libonig4
sudo pip3 install --upgrade pip setuptools requests pyzmq
sudo pip3 install --upgrade -r requirements.txt
sudo pip3 install poetry
USER=`id -u -n`
sudo chown $USER:www-data $HOME/.composer
pushd app
sudo -H -u $USER php composer.phar install --no-progress
popd
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 775 `pwd`
sudo chmod -R g+ws `pwd`/app/tmp
sudo chmod -R g+ws `pwd`/app/tmp/cache
sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
sudo chmod -R g+ws `pwd`/app/tmp/cache/models
sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/files
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
sudo chown -R $USER:www-data `pwd`
# Resque perms
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# install MySQL
sudo chmod -R 777 `pwd`/INSTALL
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
sudo chmod -R 777 `pwd`/build
sudo mkdir -p /etc/apache2/sites-available
HOST=`hostname`
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
sudo a2dissite 000-default
sudo a2ensite misp.conf
cat /etc/apache2/sites-enabled/misp.conf
sudo a2enmod rewrite
sudo systemctl restart apache2
# MISP configuration
sudo chmod -R 777 `pwd`/travis
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp travis/database.php app/Config/database.php
sudo cp app/Config/core.default.php app/Config/core.php
sudo cp app/Config/config.default.php app/Config/config.php
sudo cp travis/email.php app/Config/email.php
# Ensure the perms
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
# GPG setup
sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
sudo cp -a /dev/urandom /dev/random
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 770 `pwd`/.gnupg
# Get authkey
sudo usermod -a -G www-data $USER
- name: Configure MISP
run: |
USER=`id -u -n`
sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- name: Configure ZMQ
run: |
USER=`id -u -n`
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""'
sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1'
- name: Turn MISP live
run: |
USER=`id -u -n`
sudo -E su $USER -c 'app/Console/cake Live 1'
- name: Start workers
run: |
USER=`id -u -n`
sudo chmod +x app/Console/worker/start.sh
sudo -E su $USER -c 'app/Console/worker/start.sh &'
sleep 10
- name: Python setup
run: |
sudo chmod 777 ./key.txt
sudo chmod -R 777 ./tests
# Start workers
# Dirty install python stuff
virtualenv -p python3 ./venv
USER=`id -u -n`
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
. ./venv/bin/activate
pushd cti-python-stix2
pip install .
popd
pushd PyMISP
pip install .[fileobjects]
popd
pip install stix zmq redis plyara
deactivate
- name: Test if apache is working
run: |
HOST=`hostname`
curl http://`hostname`
AUTH=`cat key.txt`
sudo chmod -R 777 PyMISP
pushd PyMISP
echo 'url = "http://'${HOST}'"' >> tests/keys.py
echo 'key = "'${AUTH}'"' >> tests/keys.py
cat tests/keys.py
popd
- name: Run tests
run: |
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
# Ensure the perms
USER=`id -u -n`
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
AUTH=`cat key.txt`
HOST=`hostname`
pushd tests
./curl_tests_GH.sh $AUTH $HOST
popd
pushd PyMISP
git pull origin main --tags -f
poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
poetry run python tests/testlive_comprehensive.py
poetry run python tests/test_mispevent.py
popd
cp PyMISP/tests/keys.py PyMISP/examples/events/
pushd PyMISP/examples/events/
poetry run python ./create_massive_dummy_events.py -l 5 -a 30
popd
python3 tools/misp-feed/validate.py

View File

@ -148,6 +148,9 @@ before_script:
script:
- ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
- pushd tests
- ./curl_tests.sh $AUTH
- popd

View File

@ -66,7 +66,7 @@
*/
$config['CakeResque'] = array(
'Redis' => array(
'host' => 'localhost', // Redis server hostname
'host' => '127.0.0.1', // Redis server hostname
'port' => 6379, // Redis server port
'database' => 0, // Redis database number
'namespace' => 'resque', // Redis keys namespace

2
PyMISP

@ -1 +1 @@
Subproject commit 0d67babea2ba967f924978023159067aa88de2eb
Subproject commit 4e830a435b5408a01dcb8d69bc4ead6c687a221b

View File

@ -586,7 +586,7 @@ class AppController extends Controller
public function afterFilter()
{
if ($this->isApiAuthed && $this->_isRest()) {
if ($this->isApiAuthed && $this->_isRest() && $this->Session->started()) {
$this->Session->destroy();
}
}

View File

@ -110,48 +110,54 @@ class EventsController extends AppController
private function __filterOnAttributeValue($value)
{
// dissect the value
$pieces = explode('|', strtolower($value));
$include = array();
$exclude = array();
foreach ($pieces as $piece) {
if ($piece[0] === '!') {
$exclude[] = '%' . substr($piece, 1) . '%';
$includeIDs = [];
$excludeIDs = [];
if (!empty($value)) {
if (!is_array($value)) {
$pieces = explode('|', strtolower($value));
} else {
$include[] = "%$piece%";
}
}
$includeIDs = array();
if (!empty($include)) {
$includeConditions = [];
foreach ($include as $i) {
$includeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $i);
$includeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $i);
$pieces = $value;
}
$includeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => $includeConditions,
'flatten' => true,
'event_ids' => true,
'list' => true,
)));
}
$excludeIDs = array();
if (!empty($exclude)) {
$excludeConditions = [];
foreach ($exclude as $e) {
$excludeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $e);
$excludeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $e);
foreach ($pieces as $piece) {
if ($piece[0] === '!') {
$exclude[] = '%' . substr($piece, 1) . '%';
} else {
$include[] = "%$piece%";
}
}
$excludeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => $excludeConditions,
'flatten' => true,
'event_ids' => true,
'list' => true,
)));
if (!empty($include)) {
$includeConditions = [];
foreach ($include as $i) {
$includeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $i);
$includeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $i);
}
$includeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => $includeConditions,
'flatten' => true,
'event_ids' => true,
'list' => true,
)));
}
if (!empty($exclude)) {
$excludeConditions = [];
foreach ($exclude as $e) {
$excludeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $e);
$excludeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $e);
}
$excludeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => $excludeConditions,
'flatten' => true,
'event_ids' => true,
'list' => true,
)));
}
}
// return -1 as the only value in includedIDs if both arrays are empty. This will mean that no events will be shown if there was no hit
if (empty($includeIDs) && empty($excludeIDs)) {

View File

@ -1161,7 +1161,7 @@ class ServersController extends AppController
$dump = array(
'version' => $version,
'phpSettings' => $phpSettings,
'gpgStatus' => $gpgErrors[$gpgStatus],
'gpgStatus' => $gpgErrors[$gpgStatus['status']],
'proxyStatus' => $proxyErrors[$proxyStatus],
'zmqStatus' => $zmqStatus,
'stix' => $stix,

View File

@ -3978,18 +3978,17 @@ class Server extends AppModel
public function zmqAfterHook($setting, $value)
{
$pubSubTool = $this->getPubSubTool();
// If we are trying to change the enable setting to false, we don't need to test anything, just kill the server and return true.
if ($setting == 'Plugin.ZeroMQ_enable') {
if ($setting === 'Plugin.ZeroMQ_enable') {
if ($value == false || $value == 0) {
$pubSubTool->killService();
$this->getPubSubTool()->killService();
return true;
}
} elseif (!Configure::read('Plugin.ZeroMQ_enable')) {
// If we are changing any other ZeroMQ settings but the feature is disabled, don't reload the service
return true;
}
$pubSubTool->reloadServer();
$this->getPubSubTool()->reloadServer();
return true;
}
@ -5258,43 +5257,49 @@ class Server extends AppModel
return $result;
}
/**
* @param int $diagnostic_errors
* @return array
*/
public function gpgDiagnostics(&$diagnostic_errors)
{
$gpgStatus = 0;
$output = ['status' => 0, 'version' => null];
if (Configure::read('GnuPG.email') && Configure::read('GnuPG.homedir')) {
$continue = true;
try {
$gpg = GpgTool::initializeGpg();
} catch (Exception $e) {
$this->logException("Error during initializing GPG.", $e, LOG_NOTICE);
$gpgStatus = 2;
$continue = false;
$output['status'] = 2;
}
if ($continue) {
if ($output['status'] === 0) {
try {
$key = $gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
$output['version'] = $gpg->getVersion();
} catch (Exception $e) {
// ingore
}
try {
$gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
} catch (Exception $e) {
$this->logException("Error during adding GPG signing key.", $e, LOG_NOTICE);
$gpgStatus = 3;
$continue = false;
$output['status'] = 3;
}
}
if ($continue) {
if ($output['status'] === 0) {
try {
$gpgStatus = 0;
$signed = $gpg->sign('test', Crypt_GPG::SIGN_MODE_CLEAR);
$gpg->sign('test', Crypt_GPG::SIGN_MODE_CLEAR);
} catch (Exception $e) {
$this->logException("Error during GPG signing.", $e, LOG_NOTICE);
$gpgStatus = 4;
$output['status'] = 4;
}
}
} else {
$gpgStatus = 1;
$output['status'] = 1;
}
if ($gpgStatus != 0) {
if ($output['status'] !== 0) {
$diagnostic_errors++;
}
return $gpgStatus;
return $output;
}
public function zmqDiagnostics(&$diagnostic_errors)
@ -5929,7 +5934,7 @@ class Server extends AppModel
return implode('\n', $result);
}
public function update($status, &$raw = array())
public function update(array $status, &$raw = array())
{
$final = '';
$workingDirectoryPrefix = 'cd $(git rev-parse --show-toplevel) && ';
@ -5939,34 +5944,34 @@ class Server extends AppModel
);
foreach ($cleanup_commands as $cleanup_command) {
$final .= $cleanup_command . "\n\n";
$status = false;
exec($cleanup_command, $output, $status);
$returnCode = false;
exec($cleanup_command, $output, $returnCode);
$raw[] = array(
'input' => $cleanup_command,
'output' => $output,
'status' => $status
'status' => $returnCode,
);
$final .= implode("\n", $output) . "\n\n";
}
$command1 = $workingDirectoryPrefix . 'git pull origin ' . $status['branch'] . ' 2>&1';
$command2 = $workingDirectoryPrefix . 'git submodule update --init --recursive 2>&1';
$final .= $command1 . "\n\n";
$status = false;
exec($command1, $output, $status);
$returnCode = false;
exec($command1, $output, $returnCode);
$raw[] = array(
'input' => $command1,
'output' => $output,
'status' => $status
'status' => $returnCode,
);
$final .= implode("\n", $output) . "\n\n=================================\n\n";
$output = array();
$final .= $command2 . "\n\n";
$status = false;
exec($command2, $output, $status);
$returnCode = false;
exec($command2, $output, $returnCode);
$raw[] = array(
'input' => $command2,
'output' => $output,
'status' => $status
'status' => $returnCode,
);
$final .= implode("\n", $output);
return $final;

View File

@ -351,12 +351,12 @@
<p><?php echo __('This tool tests whether your GnuPG is set up correctly or not.');?></p>
<div style="background-color:#f7f7f9;width:400px;">
<?php
$colour = 'green';
$message = $gpgErrors[$gpgStatus];
if ($gpgStatus > 0) {
$colour = 'red';
$message = $gpgErrors[$gpgStatus['status']];
$color = $gpgStatus['status'] === 0 ? 'green' : 'red';
echo __('GnuPG installation and settings') . '…<span style="color:' . $color . '">' . $message . '</span><br>';
if ($gpgStatus['version']) {
echo __('GnuPG version: %s', $gpgStatus['version'] ?: __('N/A'));
}
echo __('GnuPG installation and settings') . '…<span style="color:' . $colour . ';">' . $message . '</span>';
?>
</div>
<h3><?php echo __('ZeroMQ');?></h3>

@ -1 +1 @@
Subproject commit aa499f4059bd1c14205176a3434550e7fba75a8a
Subproject commit a80da878b911e3633d88e576323e8cfec7f83e13

View File

@ -0,0 +1,15 @@
<VirtualHost %HOST%>
ServerAdmin me@me.local
ServerName %HOST%
DocumentRoot %GITHUB_WORKSPACE%/app/webroot
<Directory %GITHUB_WORKSPACE%/app/webroot>
Options -Indexes
AllowOverride all
Require all granted
</Directory>
LogLevel warn
ErrorLog /var/log/apache2/misp.local_error.log
CustomLog /var/log/apache2/misp.local_access.log combined
ServerSignature Off
</VirtualHost>

14
tests/curl_tests_GH.sh Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -e
set -x
AUTH="$1"
HOST="$2"
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events
curl -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
cat 1.csv
cut -d, -f2 --complement event.csv | sort > compare.csv
diff compare.csv 1.csv
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1