From 6e10cc37211658bba8d9cb3e1de27e822878f061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Wed, 30 Aug 2017 09:12:50 +0200 Subject: [PATCH 01/19] Removed vagrant folder --- vagrant/README.rst | 42 ----- vagrant/Vagrantfile | 128 --------------- vagrant/bootstrap.sh | 383 ------------------------------------------- 3 files changed, 553 deletions(-) delete mode 100644 vagrant/README.rst delete mode 100644 vagrant/Vagrantfile delete mode 100644 vagrant/bootstrap.sh diff --git a/vagrant/README.rst b/vagrant/README.rst deleted file mode 100644 index ab1e3a26e..000000000 --- a/vagrant/README.rst +++ /dev/null @@ -1,42 +0,0 @@ -Development environment for MISP -================================ - -Vagrant is convenient to use in order to setup your development environment. - -This VM uses `synced folders `_ -feature of Vagrant in order to let you work on the MISP source code on your -host machine while the softwares (Apache, PHP, MariaDB, etc.) and libraries -will be installed on the guest Vagrant machine. - - -Installation of VirtualBox and Vagrant --------------------------------------- - -.. code-block:: bash - - $ sudo apt-get install virtualbox vagrant - - -Deployment of MISP ------------------- - -MISP will be automatically deployed in an Ubuntu Zesty Server. - -.. code-block:: bash - - $ git clone https://github.com/MISP/MISP.git - $ cd MISP/vagrant/ - $ vagrant up - -Once the VM will be configured by Vagrant, go to the address -http://127.0.0.1:5000. - -You can now edit the source code with your favorite editor and test it in your -browser. The only thing is to not forget to restart Apache in the VM after a -modification. - -Modules activated by default in the VM: - -* `MISP galaxy `_ (http://127.0.0.1:5000/taxonomies/index) -* `MISP taxonomies `_ (http://127.0.0.1:5000/galaxies.json) -* `MISP modules `_ (curl -s http://127.0.0.1:6666/modules) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile deleted file mode 100644 index 4d8641f5d..000000000 --- a/vagrant/Vagrantfile +++ /dev/null @@ -1,128 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - - # Every Vagrant virtual environment requires a box to build off of. - #config.vm.box = "bento/ubuntu-16.04" - config.vm.box = "ubuntu/zesty64" - #config.vm.box_url = "https://atlas.hashicorp.com/ubuntu/boxes/zesty64/versions/20170412.1.0" - config.vm.provision :shell, path: "bootstrap.sh" - - # Disable automatic box update checking. If you disable this, then - # boxes will only be checked for updates when the user runs - # `vagrant box outdated`. This is not recommended. - # config.vm.box_check_update = false - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - config.vm.network :forwarded_port, guest: 80, host: 5000 - config.vm.network :forwarded_port, guest: 6666, host: 6666 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" - - # If true, then any SSH connections made will enable agent forwarding. - # Default value: false - # config.ssh.forward_agent = true - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - config.vm.synced_folder "..", "/var/www/MISP", - owner: "www-data", group: "www-data", disabled: false - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - config.vm.provider "virtualbox" do |vb| - # # Don't boot with headless mode - # vb.gui = true - # - # # Use VBoxManage to customize the VM. For example to change memory: - vb.customize ["modifyvm", :id, "--memory", "4096"] - vb.customize ["modifyvm", :id, "--name", "MISP - Ubuntu 17.04 - DEV"] - end - # - # View the documentation for the provider you're using for more - # information on available options. - - # Enable provisioning with CFEngine. CFEngine Community packages are - # automatically installed. For example, configure the host as a - # policy server and optionally a policy file to run: - # - # config.vm.provision "cfengine" do |cf| - # cf.am_policy_hub = true - # # cf.run_file = "motd.cf" - # end - # - # You can also configure and bootstrap a client to an existing - # policy server: - # - # config.vm.provision "cfengine" do |cf| - # cf.policy_server_address = "10.0.2.15" - # end - - # Enable provisioning with Puppet stand alone. Puppet manifests - # are contained in a directory path relative to this Vagrantfile. - # You will need to create the manifests directory and a manifest in - # the file default.pp in the manifests_path directory. - # - # config.vm.provision "puppet" do |puppet| - # puppet.manifests_path = "manifests" - # puppet.manifest_file = "site.pp" - # end - - # Enable provisioning with chef solo, specifying a cookbooks path, roles - # path, and data_bags path (all relative to this Vagrantfile), and adding - # some recipes and/or roles. - # - # config.vm.provision "chef_solo" do |chef| - # chef.cookbooks_path = "../my-recipes/cookbooks" - # chef.roles_path = "../my-recipes/roles" - # chef.data_bags_path = "../my-recipes/data_bags" - # chef.add_recipe "mysql" - # chef.add_role "web" - # - # # You may also specify custom JSON attributes: - # chef.json = { :mysql_password => "foo" } - # end - - # Enable provisioning with chef server, specifying the chef server URL, - # and the path to the validation key (relative to this Vagrantfile). - # - # The Opscode Platform uses HTTPS. Substitute your organization for - # ORGNAME in the URL and validation key. - # - # If you have your own Chef Server, use the appropriate URL, which may be - # HTTP instead of HTTPS depending on your configuration. Also change the - # validation key to validation.pem. - # - # config.vm.provision "chef_client" do |chef| - # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" - # chef.validation_key_path = "ORGNAME-validator.pem" - # end - # - # If you're using the Opscode platform, your validator client is - # ORGNAME-validator, replacing ORGNAME with your organization name. - # - # If you have your own Chef Server, the default validation client name is - # chef-validator, unless you changed the configuration. - # - # chef.validation_client_name = "ORGNAME-validator" -end diff --git a/vagrant/bootstrap.sh b/vagrant/bootstrap.sh deleted file mode 100644 index ba6e3403d..000000000 --- a/vagrant/bootstrap.sh +++ /dev/null @@ -1,383 +0,0 @@ -#! /usr/bin/env bash - -# Database configuration -DBHOST='localhost' -DBNAME='misp' -DBUSER_ADMIN='root' -DBPASSWORD_ADMIN="$(openssl rand -hex 32)" -DBUSER_MISP='misp' -DBPASSWORD_MISP="$(openssl rand -hex 32)" - -# Webserver configuration -PATH_TO_MISP='/var/www/MISP' -MISP_BASEURL='http://127.0.0.1:5000' -MISP_LIVE='1' -FQDN='localhost' - -# OpenSSL configuration -OPENSSL_C='LU' -OPENSSL_ST='State' -OPENSSL_L='Location' -OPENSSL_O='Organization' -OPENSSL_OU='Organizational Unit' -OPENSSL_CN='Common Name' -OPENSSL_EMAILADDRESS='info@localhost' - -# GPG configuration -GPG_REAL_NAME='Real name' -GPG_EMAIL_ADDRESS='info@localhost' -GPG_KEY_LENGTH='2048' -GPG_PASSPHRASE='' - - - - -echo -e "\n--- Installing MISP... ---\n" - - -echo -e "\n--- Updating packages list ---\n" -apt-get -qq update - - -echo -e "\n--- Install base packages ---\n" -apt-get -y install curl gcc git gnupg-agent make python openssl redis-server sudo vim zip > /dev/null 2>&1 - -# To prevent a random error when cloning with Git: 'RPC failed; curl 56 GnuTLS recv error (-54): Error in the pull function.' -git config --global http.postBuffer 1048576000 -git config --global https.postBuffer 1048576000 - -echo -e "\n--- Installing and configuring Postfix ---\n" -# # Postfix Configuration: Satellite system -# # change the relay server later with: -# sudo postconf -e 'relayhost = example.com' -# sudo postfix reload -echo "postfix postfix/mailname string `hostname`.ourdomain.org" | debconf-set-selections -echo "postfix postfix/main_mailer_type string 'Satellite system'" | debconf-set-selections -apt-get install -y postfix > /dev/null 2>&1 - - -echo -e "\n--- Installing MariaDB specific packages and settings ---\n" -apt-get install -y mariadb-client mariadb-server > /dev/null 2>&1 -# Secure the MariaDB installation (especially by setting a strong root password) -sleep 7 # give some time to the DB to launch... -apt-get install -y expect > /dev/null 2>&1 -expect -f - <<-EOF - set timeout 10 - spawn mysql_secure_installation - expect "Enter current password for root (enter for none):" - send -- "\r" - expect "Set root password?" - send -- "y\r" - expect "New password:" - send -- "${DBPASSWORD_ADMIN}\r" - expect "Re-enter new password:" - send -- "${DBPASSWORD_ADMIN}\r" - expect "Remove anonymous users?" - send -- "y\r" - expect "Disallow root login remotely?" - send -- "y\r" - expect "Remove test database and access to it?" - send -- "y\r" - expect "Reload privilege tables now?" - send -- "y\r" - expect eof -EOF -apt-get purge -y expect > /dev/null 2>&1 - - -echo -e "\n--- Installing Apache2 ---\n" -apt-get install -y apache2 apache2-doc apache2-utils > /dev/null 2>&1 -a2dismod status > /dev/null 2>&1 -a2enmod ssl > /dev/null 2>&1 -a2enmod rewrite > /dev/null 2>&1 -a2dissite 000-default > /dev/null 2>&1 -a2ensite default-ssl > /dev/null 2>&1 - - -echo -e "\n--- Installing PHP-specific packages ---\n" -apt-get install -y libapache2-mod-php php php-cli php-crypt-gpg php-dev php-json php-mysql php-opcache php-readline php-redis php-xml > /dev/null 2>&1 - - -echo -e "\n--- Restarting Apache ---\n" -systemctl restart apache2 > /dev/null 2>&1 - - -echo -e "\n--- Retrieving MISP ---\n" -mkdir $PATH_TO_MISP -chown www-data:www-data $PATH_TO_MISP -cd $PATH_TO_MISP -#git clone https://github.com/MISP/MISP.git $PATH_TO_MISP -#git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`) -git config core.filemode false -# chown -R www-data $PATH_TO_MISP -# chgrp -R www-data $PATH_TO_MISP -# chmod -R 700 $PATH_TO_MISP - - -echo -e "\n--- Installing Mitre's STIX ---\n" -apt-get install -y python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev python-setuptools > /dev/null 2>&1 -cd $PATH_TO_MISP/app/files/scripts -git clone https://github.com/CybOXProject/python-cybox.git -git clone https://github.com/STIXProject/python-stix.git -cd $PATH_TO_MISP/app/files/scripts/python-cybox -git checkout v2.1.0.12 -python setup.py install > /dev/null 2>&1 -cd $PATH_TO_MISP/app/files/scripts/python-stix -git checkout v1.1.1.4 -python setup.py install > /dev/null 2>&1 -# install mixbox to accomodate the new STIX dependencies: -cd $PATH_TO_MISP/app/files/scripts/ -git clone https://github.com/CybOXProject/mixbox.git -cd $PATH_TO_MISP/app/files/scripts/mixbox -git checkout v1.0.2 -python setup.py install > /dev/null 2>&1 - - -echo -e "\n--- Retrieving CakePHP... ---\n" -# CakePHP is included as a submodule of MISP, execute the following commands to let git fetch it: -cd $PATH_TO_MISP -git submodule init -git submodule update -# Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs: -cd $PATH_TO_MISP/app -php composer.phar require kamisama/cake-resque:4.1.2 -php composer.phar config vendor-dir Vendor -php composer.phar install -# Enable CakeResque with php-redis -phpenmod redis -# To use the scheduler worker for scheduled tasks, do the following: -cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php - - -echo -e "\n--- Setting the permissions... ---\n" -chown -R www-data:www-data $PATH_TO_MISP -chmod -R 750 $PATH_TO_MISP -chmod -R g+ws $PATH_TO_MISP/app/tmp -chmod -R g+ws $PATH_TO_MISP/app/files -chmod -R g+ws $PATH_TO_MISP/app/files/scripts/tmp - - -echo -e "\n--- Creating a database user... ---\n" -mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "create database $DBNAME;" -mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "grant usage on *.* to $DBNAME@localhost identified by '$DBPASSWORD_MISP';" -mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "grant all privileges on $DBNAME.* to '$DBUSER_MISP'@'localhost';" -mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "flush privileges;" -# Import the empty MISP database from MYSQL.sql -mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME < /var/www/MISP/INSTALL/MYSQL.sql - - -echo -e "\n--- Configuring Apache... ---\n" -# !!! apache.24.misp.ssl seems to be missing -#cp $PATH_TO_MISP/INSTALL/apache.24.misp.ssl /etc/apache2/sites-available/misp-ssl.conf -# If a valid SSL certificate is not already created for the server, create a self-signed certificate: -sudo openssl req -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=$OPENSSL_C/ST=$OPENSSL_ST/L=$OPENSSL_L/O=<$OPENSSL_O/OU=$OPENSSL_OU/CN=$OPENSSL_CN/emailAddress=$OPENSSL_EMAILADDRESS" -keyout /etc/ssl/private/misp.local.key -out /etc/ssl/private/misp.local.crt - - -echo -e "\n--- Add a VirtualHost for MISP ---\n" -cat > /etc/apache2/sites-available/misp-ssl.conf < - ServerAdmin me@me.local - ServerName misp.local - DocumentRoot $PATH_TO_MISP/app/webroot - - - Options -Indexes - AllowOverride all - Require all granted - - - LogLevel warn - ErrorLog /var/log/apache2/misp.local_error.log - CustomLog /var/log/apache2/misp.local_access.log combined - ServerSignature Off - -EOF -# cat > /etc/apache2/sites-available/misp-ssl.conf < -# ServerName misp.local -# -# Redirect permanent / https://$FQDN -# -# LogLevel warn -# ErrorLog /var/log/apache2/misp.local_error.log -# CustomLog /var/log/apache2/misp.local_access.log combined -# ServerSignature Off -# -# -# -# ServerAdmin me@me.local -# ServerName misp.local -# DocumentRoot $PATH_TO_MISP/app/webroot -# -# -# Options -Indexes -# AllowOverride all -# Require all granted -# -# -# SSLEngine On -# SSLCertificateFile /etc/ssl/private/misp.local.crt -# SSLCertificateKeyFile /etc/ssl/private/misp.local.key -# #SSLCertificateChainFile /etc/ssl/private/misp-chain.crt -# -# LogLevel warn -# ErrorLog /var/log/apache2/misp.local_error.log -# CustomLog /var/log/apache2/misp.local_access.log combined -# ServerSignature Off -# -# EOF -# activate new vhost -a2dissite default-ssl -a2ensite misp-ssl - - -echo -e "\n--- Restarting Apache ---\n" -systemctl restart apache2 > /dev/null 2>&1 - - -echo -e "\n--- Configuring log rotation ---\n" -cp $PATH_TO_MISP/INSTALL/misp.logrotate /etc/logrotate.d/misp - - -echo -e "\n--- MISP configuration ---\n" -# There are 4 sample configuration files in /var/www/MISP/app/Config that need to be copied -cp -a $PATH_TO_MISP/app/Config/bootstrap.default.php /var/www/MISP/app/Config/bootstrap.php -cp -a $PATH_TO_MISP/app/Config/database.default.php /var/www/MISP/app/Config/database.php -cp -a $PATH_TO_MISP/app/Config/core.default.php /var/www/MISP/app/Config/core.php -cp -a $PATH_TO_MISP/app/Config/config.default.php /var/www/MISP/app/Config/config.php -cat > $PATH_TO_MISP/app/Config/database.php < 'Database/Mysql', - //'datasource' => 'Database/Postgres', - 'persistent' => false, - 'host' => '$DBHOST', - 'login' => '$DBUSER_MISP', - 'port' => 3306, // MySQL & MariaDB - //'port' => 5432, // PostgreSQL - 'password' => '$DBPASSWORD_MISP', - 'database' => '$DBNAME', - 'prefix' => '', - 'encoding' => 'utf8', - ); -} -EOF -# and make sure the file permissions are still OK -chown -R www-data:www-data $PATH_TO_MISP/app/Config -chmod -R 750 $PATH_TO_MISP/app/Config -# Set some MISP directives with the command line tool -$PATH_TO_MISP/app/Console/cake Baseurl $MISP_BASEURL -$PATH_TO_MISP/app/Console/cake Live $MISP_LIVE - - -echo -e "\n--- Generating a GPG encryption key... ---\n" -apt-get install -y rng-tools haveged -mkdir $PATH_TO_MISP/.gnupg -chmod 700 $PATH_TO_MISP/.gnupg -cat >gen-key-script < $PATH_TO_MISP/app/webroot/gpg.asc - - -echo -e "\n--- Making the background workers start on boot... ---\n" -chmod 755 $PATH_TO_MISP/app/Console/worker/start.sh -cat > /etc/systemd/system/workers.service < /dev/null -systemctl restart workers.service > /dev/null - - -echo -e "\n--- Installing MISP modules... ---\n" -apt-get install -y python3-dev python3-pip libpq5 libjpeg-dev > /dev/null 2>&1 -cd /usr/local/src/ -git clone https://github.com/MISP/misp-modules.git -cd misp-modules -pip3 install -I -r REQUIREMENTS > /dev/null 2>&1 -pip3 install -I . > /dev/null 2>&1 -cat > /etc/systemd/system/misp-modules.service < /dev/null -systemctl restart misp-modules.service > /dev/null - - -echo -e "\n--- Restarting Apache... ---\n" -systemctl restart apache2 > /dev/null 2>&1 -sleep 5 - -echo -e "\n--- Updating the galaxies... ---\n" -sudo -E $PATH_TO_MISP/app/Console/cake userInit -q > /dev/null -AUTH_KEY=$(mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e "SELECT authkey FROM users;" | tail -1) -curl -k -X POST -H "Authorization: $AUTH_KEY" -H "Accept: application/json" -v http://127.0.0.1/galaxies/update > /dev/null 2>&1 - - -echo -e "\n--- Updating the taxonomies... ---\n" -curl -k -X POST -H "Authorization: $AUTH_KEY" -H "Accept: application/json" -v http://127.0.0.1/taxonomies/update > /dev/null 2>&1 - - -# echo -e "\n--- Enabling MISP new pub/sub feature (ZeroMQ)... ---\n" -# # ZeroMQ depends on the Python client for Redis -# pip install redis > /dev/null 2>&1 -# ## Install ZeroMQ and prerequisites -# apt-get install -y pkg-config > /dev/null 2>&1 -# cd /usr/local/src/ -# git clone git://github.com/jedisct1/libsodium.git > /dev/null 2>&1 -# cd libsodium -# /autogen.sh > /dev/null 2>&1 -# ./configure > /dev/null 2>&1 -# make check > /dev/null 2>&1 -# make > /dev/null 2>&1 -# make install > /dev/null 2>&1 -# ldconfig > /dev/null 2>&1 -# cd /usr/local/src/ -# wget https://archive.org/download/zeromq_4.1.5/zeromq-4.1.5.tar.gz > /dev/null 2>&1 -# tar -xvf zeromq-4.1.5.tar.gz > /dev/null 2>&1 -# cd zeromq-4.1.5/ -# ./autogen.sh > /dev/null 2>&1 -# ./configure > /dev/null 2>&1 -# make check > /dev/null 2>&1 -# make > /dev/null 2>&1 -# make install > /dev/null 2>&1 -# ldconfig > /dev/null 2>&1 -# ## install pyzmq -# pip install pyzmq > /dev/null 2>&1 - - -echo -e "\e[32mMISP is ready\e[0m" -echo -e "\e[0mPoint your Web browser to \e[33m$MISP_BASEURL\e[0m" -echo -e "\e[0mDefault user/pass = \e[33madmin@admin.test/admin\e[0m" From 4584240517af7a8404d2fc62b9290352fdeb04bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Wed, 30 Aug 2017 09:17:19 +0200 Subject: [PATCH 02/19] added misp-vagrant module --- .gitmodules | 3 +++ misp-vagrant | 1 + 2 files changed, 4 insertions(+) create mode 160000 misp-vagrant diff --git a/.gitmodules b/.gitmodules index db318e1f2..c308eedbd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,3 +21,6 @@ [submodule "app/files/misp-galaxy"] path = app/files/misp-galaxy url = https://github.com/MISP/misp-galaxy +[submodule "misp-vagrant"] + path = misp-vagrant + url = https://github.com/MISP/misp-vagrant.git diff --git a/misp-vagrant b/misp-vagrant new file mode 160000 index 000000000..8622be9f3 --- /dev/null +++ b/misp-vagrant @@ -0,0 +1 @@ +Subproject commit 8622be9f3074ead2e315803aac94eec6f16461b5 From 4730549f7ca68862704015a5d302da884d9c0cdf Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 5 Sep 2017 16:07:59 +0200 Subject: [PATCH 03/19] fix: Fixed the favourite tags not showing up in the tag index --- app/Controller/TagsController.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index d7c91f540..ce39737be 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -121,8 +121,12 @@ class TagsController extends AppController { } unset($paginated[$k]['AttributeTag']); if (!empty($tag['FavouriteTag'])) { - foreach ($tag['FavouriteTag'] as $ft) if ($ft['user_id'] == $this->Auth->user('id')) $paginated[$k]['Tag']['favourite'] = true; - if (!isset($tag['Tag']['favourite'])) $paginated[$k]['Tag']['favourite'] = false; + foreach ($tag['FavouriteTag'] as $ft) { + if ($ft['user_id'] == $this->Auth->user('id')) { + $paginated[$k]['Tag']['favourite'] = true; + } + } + if (!isset($paginated[$k]['Tag']['favourite'])) $paginated[$k]['Tag']['favourite'] = false; } else $paginated[$k]['Tag']['favourite'] = false; unset($paginated[$k]['FavouriteTag']); if (!empty($taxonomyNamespaces)) { From 839876733b0ae7aa2727063e5954dea23a71b97d Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 6 Sep 2017 16:38:36 +0200 Subject: [PATCH 04/19] fix: MISP galaxy updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 6bb2a0738..146f8fd1a 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 6bb2a0738365c3b62154e9ccd2adccc012a64a35 +Subproject commit 146f8fd1abde947f9f277205ddc5f9392d545527 From bf2d7f920b2b3fc41c1ec3cecd347b8eb9e848e4 Mon Sep 17 00:00:00 2001 From: RT Hatfield Date: Wed, 6 Sep 2017 11:56:58 -0400 Subject: [PATCH 05/19] Fixing bug in feed-fetch sched. task --- app/Console/Command/ServerShell.php | 9 ++------- app/Controller/TasksController.php | 24 ------------------------ 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 67dd3d468..a353ccbd6 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -197,14 +197,13 @@ class ServerShell extends AppShell if ($timestamp != $task['Task']['next_execution_time']) { return; } - if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueCachePull', $userId, $taskId, $action); + if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueFeed', $userId, $taskId, $action); $user = $this->User->getAuthUser($userId); - $count = count($feeds); $failCount = 0; if ($action == 0) { $feeds = $this->Feed->find('all', array( 'recursive' => -1, - 'conditions' => array('enabled' => 1) + 'conditions' => array('enabled' => true) )); foreach ($feeds as $k => $feed) { $this->Job->create(); @@ -220,10 +219,8 @@ class ServerShell extends AppShell ); $this->Job->save($data); $jobId = $this->Job->id; - App::uses('SyncTool', 'Tools'); $result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId); $this->Job->save(array( - 'id' => $jobId, 'message' => 'Job done.', 'progress' => 100, 'status' => 4 @@ -240,7 +237,6 @@ class ServerShell extends AppShell $data = array( 'worker' => 'default', 'job_type' => 'feed_cache', - 'job_input' => 'Feed: ' . $feed['Feed']['id'], 'retries' => 0, 'org' => $user['Organisation']['name'], 'org_id' => $user['org_id'], @@ -251,7 +247,6 @@ class ServerShell extends AppShell $jobId = $this->Job->id; $result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all'); $this->Job->save(array( - 'id' => $jobId, 'message' => 'Job done.', 'progress' => 100, 'status' => 4 diff --git a/app/Controller/TasksController.php b/app/Controller/TasksController.php index d1e5b1a2b..b0fbaaf04 100644 --- a/app/Controller/TasksController.php +++ b/app/Controller/TasksController.php @@ -133,28 +133,4 @@ class TasksController extends AppController { $this->Task->saveField('process_id', $process_id); } - private function _feedPullScheduler($timestamp, $id) { - $process_id = CakeResque::enqueueAt( - $timestamp, - 'default', - 'ServerShell', - array('enqueuePull', $timestamp, $this->Auth->user('id'), $id), - true - ); - $this->Task->id = $id; - $this->Task->saveField('process_id', $process_id); - } - - private function _feedCacheScheduler($timestamp, $id) { - $process_id = CakeResque::enqueueAt( - $timestamp, - 'default', - 'ServerShell', - array('enqueuePull', $timestamp, $this->Auth->user('id'), $id), - true - ); - $this->Task->id = $id; - $this->Task->saveField('process_id', $process_id); - } - } From 423060111639801c423ad7fa59e2ed8de8e3f520 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 6 Sep 2017 23:26:21 +0200 Subject: [PATCH 06/19] fix: Fixes to various issues with the cert auth --- app/Model/User.php | 1 + .../Auth/CertificateAuthenticate.php | 124 +++++++----------- 2 files changed, 49 insertions(+), 76 deletions(-) diff --git a/app/Model/User.php b/app/Model/User.php index a99348332..6eb37b3db 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -583,6 +583,7 @@ class User extends AppModel { // get the current user and rearrange it to be in the same format as in the auth component public function getAuthUser($id) { + if (empty($id)) throw new Exception('Invalid user ID.'); $conditions = array('User.id' => $id); $user = $this->find('first', array('conditions' => $conditions, 'recursive' => -1,'contain' => array('Organisation', 'Role', 'Server'))); if (empty($user)) return $user; diff --git a/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php b/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php index ae04ae02e..43428ba3e 100644 --- a/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php +++ b/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php @@ -127,97 +127,69 @@ class CertificateAuthenticate extends BaseAuthenticate // to enable stateless authentication public function getUser(CakeRequest $request) { - if (is_null(self::$user)) { + if (empty(self::$user)) { if (self::$client) { self::$user = self::$client; - + // If $sync is true, allow the creation of the user from the certificate $sync = Configure::read('CertAuth.syncUser'); - if ($sync) { self::getRestUser(); } // find and fill user with model - $cn = Configure::read('CertAuth.userModel'); - if ($cn) { - $k = Configure::read('CertAuth.userModelKey'); - if ($k) { - $q = array($k=>self::$user[$k]); - } else { - $q = self::$user; - } - $User = ClassRegistry::init($cn); - $U = $User->find('first', array( - 'conditions' => $q, - 'recursive' => false - )); - if ($U) { - if ($sync) { - $write = array(); - - if (!isset(self::$user['org_id']) && isset(self::$user['org'])) { - self::$user['org_id']=$User->Organisation->createOrgFromName(self::$user['org'], $User->id, true); - // reset user defaults in case it's a different org_id - if(self::$user['org_id'] && $U[$cn]['org_id']!=self::$user['org_id']) { - $d = Configure::read('CertAuth.userDefaults'); - if ($d && is_array($d)) { - self::$user = $d + self::$user; - } - unset($d); - } - unset(self::$user['org']); - } - - foreach (self::$user as $k=>$v) { - if (array_key_exists($k, $U[$cn]) && trim($U[$cn][$k])!=trim($v)) { - $write[] = $k; - $U[$cn][$k] = trim($v); - } - unset($k, $v); - } - if ($write && !$User->save($U[$cn], true, $write)) { - CakeLog::write('alert', 'Could not update model at database with RestAPI data.'); - } - unset($write); - } - self::$user = $User->getAuthUser($U[$cn]['id']); - if (isset(self::$user['gpgkey'])) unset(self::$user['gpgkey']); - } else if ($sync && self::$user) { - $User->create(); - $org=null; + $userModelKey = empty(Configure::read('CertAuth.userModelKey')) ? 'email' : Configure::read('CertAuth.userModelKey'); + $userDefaults = Configure::read('CertAuth.userDefaults'); + $this->User = ClassRegistry::init('User'); + $existingUser = $this->User->find('first', array( + 'conditions' => array($userModelKey => self::$user[$userModelKey]), + 'recursive' => false + )); + if ($existingUser) { + if ($sync) { if (!isset(self::$user['org_id']) && isset(self::$user['org'])) { - $org = self::$user['org']; + self::$user['org_id'] = $this->User->Organisation->createOrgFromName(self::$user['org'], $existingUser['User']['id'], true); + // reset user defaults in case it's a different org_id + if (self::$user['org_id'] && $existingUser['User']['org_id'] != self::$user['org_id']) { + if ($userDefaults && is_array($userDefaults)) { + self::$user = array_merge($userDefaults + self::$user); + } + } unset(self::$user['org']); } - - $d = Configure::read('CertAuth.userDefaults'); - if ($d && is_array($d)) { - self::$user += $d; - } - unset($d); - - if ($User->save(self::$user, true)) { - $id = $User->id; - if ($org) { - self::$user['id'] = $id; - self::$user['org_id']=$User->Organisation->createOrgFromName($org, $User->id, true); - $User->save(self::$user, true, array('org_id')); + $write = array(); + foreach (self::$user as $k => $v) { + if (isset($existingUser['User'][$k]) && trim($existingUser['User'][$k]) != trim($v)) { + $write[] = $k; + $existingUser['User'][$k] = trim($v); } - - self::$user = $User->getAuthUser($id); - unset($id); - if (isset(self::$user['gpgkey'])) unset(self::$user['gpgkey']); - } else { - CakeLog::write('alert', 'Could not insert model at database from RestAPI data.'); } - unset($org); - } else { - // No match -- User doesn't exist !!! - self::$user = false; + if (!empty($write) && !$this->User->save($existingUser['User'], true, $write)) { + CakeLog::write('alert', 'Could not update model at database with RestAPI data.'); + } } - unset($U, $User, $q, $k); + self::$user = $this->User->getAuthUser($existingUser['User']['id']); + if (isset(self::$user['gpgkey'])) unset(self::$user['gpgkey']); + } else if ($sync && !empty(self::$user)) { + $org=null; + if (!isset(self::$user['org_id']) && isset(self::$user['org'])) { + self::$user['org_id'] = $this->User->Organisation->createOrgFromName($org, 0, true); + unset(self::$user['org']); + } + if ($userDefaults && is_array($userDefaults)) { + self::$user = array_merge(self::$user, $userDefaults); + } + $this->User->create(); + if ($this->User->save(self::$user)) { + $id = $this->User->id; + self::$user = $this->User->getAuthUser($id); + if (isset(self::$user['gpgkey'])) unset(self::$user['gpgkey']); + } else { + CakeLog::write('alert', 'Could not insert model at database from RestAPI data. Reason: ' . json_encode($this->User->validationErrors)); + } + } else { + // No match -- User doesn't exist !!! + self::$user = false; } - unset($cn); } } From 9c9048422a13d32d658e4f03540389344f409a29 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 7 Sep 2017 00:06:02 +0200 Subject: [PATCH 07/19] fix: fixes various issues with the certauth --- .../Controller/Component/Auth/CertificateAuthenticate.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php b/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php index 43428ba3e..ce3d18db4 100644 --- a/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php +++ b/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php @@ -170,7 +170,8 @@ class CertificateAuthenticate extends BaseAuthenticate self::$user = $this->User->getAuthUser($existingUser['User']['id']); if (isset(self::$user['gpgkey'])) unset(self::$user['gpgkey']); } else if ($sync && !empty(self::$user)) { - $org=null; + $org = isset(self::$client['org']) ? self::$client['org'] : null; + if ($org == null) return false; if (!isset(self::$user['org_id']) && isset(self::$user['org'])) { self::$user['org_id'] = $this->User->Organisation->createOrgFromName($org, 0, true); unset(self::$user['org']); @@ -192,7 +193,6 @@ class CertificateAuthenticate extends BaseAuthenticate } } } - return self::$user; } From e61aefd64d1a8586c54f1979f2020284129d4d86 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 7 Sep 2017 12:16:50 +0200 Subject: [PATCH 08/19] fix: Fix to an issue blocking the JSON download of single events --- app/Controller/EventsController.php | 8 ++++++-- app/Model/Event.php | 14 +++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 6e07188a6..a320b0142 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2630,7 +2630,7 @@ class EventsController extends AppController { if (isset($searchall) && ($searchall == 1 || $searchall === true || $searchall == 'true')) { $eventIds = $this->__quickFilter($value); } else { - $parameters = array('value', 'type', 'category', 'org', 'eventid', 'uuid'); + $parameters = array('value', 'type', 'category', 'org', 'uuid', 'eventid'); $attributeLevelFilters = array('value', 'type', 'category', 'uuid'); $preFilterLevel = 'event'; foreach ($parameters as $k => $param) { @@ -2639,6 +2639,10 @@ class EventsController extends AppController { $preFilterLevel = 'attribute'; } $conditions = $this->Event->setSimpleConditions($parameters[$k], ${$parameters[$k]}, $conditions); + if ($param == 'eventid') { + $restrictScopeToEvents = true; + } + $conditions = $this->Event->setSimpleConditions($parameters[$k], ${$parameters[$k]}, $conditions, !empty($restrictScopeToEvents)); } } // If we sent any tags along, load the associated tag names for each attribute @@ -2653,7 +2657,7 @@ class EventsController extends AppController { $params = array( 'conditions' => $conditions ); - $eventIds = $this->Event->fetchSipleEventIds($this->Auth->user(), $params); + $eventIds = $this->Event->fetchSimpleEventIds($this->Auth->user(), $params); } else { $params = array( 'conditions' => $conditions, diff --git a/app/Model/Event.php b/app/Model/Event.php index c8d2e3fff..b31781b63 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3358,7 +3358,7 @@ class Event extends AppModel { return array('data' => array(), 'csv' => array()); } - public function setSimpleConditions($parameterKey, $parameterValue, $conditions) { + public function setSimpleConditions($parameterKey, $parameterValue, $conditions, $restrictScopeToEvents = false) { if (is_array($parameterValue)) { $elements = $parameterValue; } else { @@ -3386,7 +3386,11 @@ class Event extends AppModel { $subcondition['AND'][] = array('Event.orgc_id !=' => $o['Org']['id']); } } else if ($parameterKey === 'eventid') { - $subcondition['AND'][] = array('Attribute.event_id !=' => substr($v, 1)); + if ($restrictScopeToEvents) { + $subcondition['AND'][] = array('Event.id !=' => substr($v, 1)); + } else { + $subcondition['AND'][] = array('Attribute.event_id !=' => substr($v, 1)); + } } else if ($parameterKey === 'uuid') { $subcondition['AND'][] = array('Event.uuid !=' => substr($v, 1)); $subcondition['AND'][] = array('Attribute.uuid !=' => substr($v, 1)); @@ -3411,7 +3415,11 @@ class Event extends AppModel { $subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']); } } else if ($parameterKey === 'eventid') { - $subcondition['OR'][] = array('Attribute.event_id' => $v); + if ($restrictScopeToEvents) { + $subcondition['OR'][] = array('Event.id' => $v); + } else { + $subcondition['OR'][] = array('Attribute.event_id' => $v); + } } else if ($parameterKey === 'uuid') { $subcondition['OR'][] = array('Attribute.uuid' => $v); $subcondition['OR'][] = array('Event.uuid' => $v); From ef9b9ba1885ff2d7ddc1f52e60bc28cb40d1cb01 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 7 Sep 2017 12:19:39 +0200 Subject: [PATCH 09/19] fix: Corrected a copy paste mistake --- app/Controller/EventsController.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index a320b0142..32c661586 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2638,7 +2638,6 @@ class EventsController extends AppController { if (in_array($param, $attributeLevelFilters)) { $preFilterLevel = 'attribute'; } - $conditions = $this->Event->setSimpleConditions($parameters[$k], ${$parameters[$k]}, $conditions); if ($param == 'eventid') { $restrictScopeToEvents = true; } From eef0f84ec347fe4f0507e8282e9f8d0cafeb1964 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 7 Sep 2017 15:30:10 +0200 Subject: [PATCH 10/19] fix: Added better debugging to the password shell --- app/Console/Command/PasswordShell.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Console/Command/PasswordShell.php b/app/Console/Command/PasswordShell.php index 94e9a1344..b4b3b77f9 100644 --- a/app/Console/Command/PasswordShell.php +++ b/app/Console/Command/PasswordShell.php @@ -14,12 +14,16 @@ class PasswordShell extends AppShell { else { // get the users that need their password hashed $results = $this->User->find('first', array('conditions' => array('email' => $this->args[0]), 'recursive' => -1)); + if (empty($results)) { + echo 'User not found. Make sure you use the correct syntax: /var/www/MISP/app/Console/cake Password [email] [password]' . PHP_EOL; + exit; + } $results['User']['password'] = $this->args[1]; $results['User']['confirm_password'] = $this->args[1]; $results['User']['change_pw'] = 1; if (!$this->User->save($results)) { echo 'Could not update account for User.id = ', $results['User']['id'], PHP_EOL; - debug($this->User->validationErrors); + echo json_encode($this->User->validationErrors) . PHP_EOL; $this->out(print_r($this->User->invalidFields(), true)); } echo 'Updated ', PHP_EOL; From be111a470204a974c50682054c9c7d4b94396ed9 Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 8 Sep 2017 14:25:36 +0200 Subject: [PATCH 11/19] fix: Fix to certauth pains --- .../Component/Auth/CertificateAuthenticate.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php b/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php index ce3d18db4..1ad5c481f 100644 --- a/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php +++ b/app/Plugin/CertAuth/Controller/Component/Auth/CertificateAuthenticate.php @@ -133,17 +133,19 @@ class CertificateAuthenticate extends BaseAuthenticate // If $sync is true, allow the creation of the user from the certificate $sync = Configure::read('CertAuth.syncUser'); if ($sync) { - self::getRestUser(); + if (!self::getRestUser()) return false; } // find and fill user with model $userModelKey = empty(Configure::read('CertAuth.userModelKey')) ? 'email' : Configure::read('CertAuth.userModelKey'); $userDefaults = Configure::read('CertAuth.userDefaults'); $this->User = ClassRegistry::init('User'); - $existingUser = $this->User->find('first', array( - 'conditions' => array($userModelKey => self::$user[$userModelKey]), - 'recursive' => false - )); + if (!empty(self::$user[$userModelKey])) { + $existingUser = $this->User->find('first', array( + 'conditions' => array($userModelKey => self::$user[$userModelKey]), + 'recursive' => false + )); + } if ($existingUser) { if ($sync) { if (!isset(self::$user['org_id']) && isset(self::$user['org'])) { From 1489cd34cf5621827c29187e1729393e6bb096c7 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 11 Sep 2017 11:51:37 +0200 Subject: [PATCH 12/19] Security vulnerability reporting about "high number of published CVEs vs a few swept under the rug" --- CONTRIBUTING.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c61c131d4..c022d60b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,14 +3,20 @@ MISP project is a large free software project composed of multiple sub-projects which are contributed by different contributors who are generally active users of the MISP project. MISP project fully supports the [Contributor Covenant Code of Conduct](https://github.com/MISP/MISP/blob/2.4/code_of_conduct.md) to foster an open and dynamic environment for contributing and the exchange in the threat intelligence and information exchange field. +The MISP roadmap is mostly based on the user communities (e.g. private communities, CSIRTs communities, security researchers, ISACs - Information Sharing and Analysis Center, security providers, governmental or military organisations) relying on MISP to perform their duties of information sharing and analysis. If you see an existing issue which covers your needs, don't hesitate to comment on it. + Participating in the MISP project is easy and everyone can contribute following their own ability: -- Reporting a bug or an issue, suggesting a new feature in the [MISP core](https://github.com/MISP/MISP/issues), [misp-modules](https://github.com/MISP/misp-modules/issues), [misp-book](https://github.com/MISP/misp-book/issues), [misp-taxonomies](https://github.com/MISP/misp-taxonomies/issues), [misp-galaxy](https://github.com/MISP/misp-galaxy/issues) or any of the projects within the [MISP project org](https://github.com/MISP/). Don't hesitate to add as much information as you can, including the version of MISP which you are running, screen-shots with annotation, suggested features and steps on how to reproduce an issue. Don't hesitate to search the existing open issues and comment on existing ones. This is an indicator for us regarding the priority of certain features +## Reporting a bug or an issue, suggesting features + +Reporting a bug or an issue, suggesting a new feature in the [MISP core](https://github.com/MISP/MISP/issues), [misp-modules](https://github.com/MISP/misp-modules/issues), [misp-book](https://github.com/MISP/misp-book/issues), [misp-taxonomies](https://github.com/MISP/misp-taxonomies/issues), [misp-galaxy](https://github.com/MISP/misp-galaxy/issues) or any of the projects within the [MISP project org](https://github.com/MISP/). Don't hesitate to add as much information as you can, including the version of MISP which you are running, screen-shots with annotation, suggested features and steps on how to reproduce an issue. Don't hesitate to search the existing open issues and comment on existing ones. This is an indicator for us regarding the priority of certain features and how important these are to the users. -- Reporting security vulnerabilities is of great importance for us, as MISP is used in multiple critical infrastructures. In the case of a security vulnerability report, we ask the reporter to directly report to [CIRCL](https://www.circl.lu/contact/) encrypting the report with the PGP key: CA57 2205 C002 4E06 BA70 BE89 EAAD CFFC 22BD 4CD5. We usually fix reported and confirmed security vulnerabilities in less than 48 hours, followed by a software release containing the fixes within the following days. If you report security vulnerabilities, don't forget to tell us if and how you want to be acknowledged and if you already requested CVE(s). If not, we will request the CVE directly. +## Reporting security vulnerabilities -The MISP roadmap is mostly based on the user communities (e.g. private communities, CSIRTs communities, security researchers, ISACs - Information Sharing and Analysis Center, security providers, governmental or military organisations) relying on MISP to perform their duties of information sharing and analysis. If you see an existing issue which covers your needs, don't hesitate to comment on it. +Reporting security vulnerabilities is of great importance for us, as MISP is used in multiple critical infrastructures. In the case of a security vulnerability report, we ask the reporter to directly report to [CIRCL](https://www.circl.lu/contact/), encrypting the report with the PGP key: CA57 2205 C002 4E06 BA70 BE89 EAAD CFFC 22BD 4CD5. We usually fix reported and confirmed security vulnerabilities in less than 48 hours, followed by a software release containing the fixes within the following days. If you report security vulnerabilities, don't forget to tell us if and how you want to be acknowledged and if you already requested CVE(s). If not, we will request the CVE directly. + +As one of the critical user-bases of MISP consists of the CSIRT community, it is our duty to clearly state which bug could be potentially abused and could have a security impact on a deployed MISP instance. CVE assignment is performed even for minor bugs having some possible security impact. This allows users using MISP instances in their environment to understand which bugs could have an impact on their security. We firmly believe that, even though unfortunately it is often not regarded as common practice in our industry, being as transparent as possible about vulnerabilities, no matter how minor, is of absolute crucial importance. At MISP-project, we care about the security of our users and prefer to have a high number of published CVEs than to a few swept under the rug. ## MISP core From 406a86826a00d9468ff610a3cde5a7c462b7936c Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 11 Sep 2017 14:21:16 +0200 Subject: [PATCH 13/19] fix: capitalisation of default tlp tag didn't match the ones coming from taxonomies in the event alert e-mail subject --- app/Model/Event.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index b31781b63..acf1ce28b 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1752,7 +1752,7 @@ class Event extends AppModel { } else { $subject = ''; } - $subjMarkingString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber"; + $subjMarkingString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber"; $subjTag = !empty(Configure::read('MISP.email_subject_tag')) ? Configure::read('MISP.email_subject_tag') : "tlp"; $tagLen = strlen($subjTag); foreach ($event[0]['EventTag'] as $k => $tag) { From a931af72238d70cc2b9242bb4e77463469ff937c Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 12 Sep 2017 10:20:38 +0200 Subject: [PATCH 14/19] chg: Some tuning to the freetext import tool --- app/Lib/Tools/ComplexTypeTool.php | 8 ++++++-- app/View/Events/resolved_attributes.ctp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index c2204b280..c276ef8e0 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -10,7 +10,8 @@ class ComplexTypeTool { '/\[dot\]/' => '.', '/\(dot\)/' => '.', '/\\\\\./' => '.', - '/\.+/' => '.' + '/\.+/' => '.', + '/\[hxxp:\/\/\]/' => 'http://' ); private $__tlds = array(); @@ -287,7 +288,10 @@ class ComplexTypeTool { } private function __resolveFilename($input) { - if ((preg_match('/^.:/', $input) || strpos($input, '.') !=0)) return true; + if ((preg_match('/^.:/', $input) || strpos($input, '.') !=0)) { + $parts = explode('.', $input); + if (!is_numeric($parts[count($parts)-1]) && ctype_alnum($parts[count($parts)-1])) return true; + } return false; } diff --git a/app/View/Events/resolved_attributes.ctp b/app/View/Events/resolved_attributes.ctp index 8355df612..840de0bb8 100644 --- a/app/View/Events/resolved_attributes.ctp +++ b/app/View/Events/resolved_attributes.ctp @@ -70,7 +70,7 @@ echo $this->Form->input('Attribute' . $k . 'Value', array( 'label' => false, 'value' => $item['value'], - 'style' => 'padding:0px;height:20px;margin-bottom:0px;width:90%;min-width:200px;', + 'style' => 'padding:0px;height:20px;margin-bottom:0px;width:90%;min-width:400px;', 'div' => false )); ?> From 2e47a630a551ab152a98df1ea743d818718b68d2 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 13 Sep 2017 15:28:56 +0200 Subject: [PATCH 15/19] fix: Avoid compatibility issue with AGPL license and its warranty clause --- app/View/Events/nids.ctp | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/View/Events/nids.ctp b/app/View/Events/nids.ctp index f51607df4..aa61dedd7 100644 --- a/app/View/Events/nids.ctp +++ b/app/View/Events/nids.ctp @@ -1,6 +1,4 @@ Date: Wed, 13 Sep 2017 09:35:24 -0400 Subject: [PATCH 16/19] fix for issue #2458 --- app/Console/Command/ServerShell.php | 126 ++++++++++++++-------------- app/Controller/TasksController.php | 8 +- app/Model/Task.php | 4 +- 3 files changed, 73 insertions(+), 65 deletions(-) diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index a353ccbd6..1614b6ba1 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -185,79 +185,81 @@ class ServerShell extends AppShell $this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . $count); } - public function enqueueFeed() { + public function enqueueFeedFetch() { $timestamp = $this->args[0]; $userId = $this->args[1]; $taskId = $this->args[2]; - // action options: - // 0 = pull - // 1 = cache - $action = $this->args[3]; $task = $this->Task->read(null, $taskId); if ($timestamp != $task['Task']['next_execution_time']) { return; } - if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueFeed', $userId, $taskId, $action); + if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueFeedFetch', $userId, $taskId); $user = $this->User->getAuthUser($userId); $failCount = 0; - if ($action == 0) { - $feeds = $this->Feed->find('all', array( - 'recursive' => -1, - 'conditions' => array('enabled' => true) - )); - foreach ($feeds as $k => $feed) { - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'feed_pull', - 'job_input' => 'Feed: ' . $feed['Feed']['id'], - 'retries' => 0, - 'org' => $user['Organisation']['name'], - 'org_id' => $user['org_id'], - 'process_id' => 'Part of scheduled feed pull', - 'message' => 'Pulling.', - ); - $this->Job->save($data); - $jobId = $this->Job->id; - $result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId); - $this->Job->save(array( - 'message' => 'Job done.', - 'progress' => 100, - 'status' => 4 - )); - if ($result !== true) { - $this->Job->saveField('message', 'Could not pull feed.'); - $failCount++; - } - } - $this->Task->id = $task['Task']['id']; - $this->Task->saveField('message', count($feeds) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . count($feeds)); - } else { - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'feed_cache', - 'retries' => 0, - 'org' => $user['Organisation']['name'], - 'org_id' => $user['org_id'], - 'process_id' => 'Part of scheduled feed caching', - 'message' => 'Caching.', - ); - $this->Job->save($data); - $jobId = $this->Job->id; - $result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all'); - $this->Job->save(array( - 'message' => 'Job done.', - 'progress' => 100, - 'status' => 4 - )); - $this->Task->id = $task['Task']['id']; - $this->Task->saveField('message', 'Job completed at ' . date('d/m/Y - H:i:s')); - } + $feeds = $this->Feed->find('all', array( + 'recursive' => -1, + 'conditions' => array('enabled' => true) + )); + foreach ($feeds as $k => $feed) { + $this->Job->create(); + $data = array( + 'worker' => 'default', + 'job_type' => 'feed_fetch', + 'job_input' => 'Feed: ' . $feed['Feed']['id'], + 'retries' => 0, + 'org' => $user['Organisation']['name'], + 'org_id' => $user['org_id'], + 'process_id' => 'Part of scheduled feed fetch', + 'message' => 'Pulling.', + ); + $this->Job->save($data); + $jobId = $this->Job->id; + $result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId); + $this->Job->save(array( + 'message' => 'Job done.', + 'progress' => 100, + 'status' => 4 + )); + if ($result !== true) { + $this->Job->saveField('message', 'Could not fetch feed.'); + $failCount++; + } + } + $this->Task->id = $task['Task']['id']; + $this->Task->saveField('message', count($feeds) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . count($feeds)); } - - + public function enqueueFeedCache() { + $timestamp = $this->args[0]; + $userId = $this->args[1]; + $taskId = $this->args[2]; + $task = $this->Task->read(null, $taskId); + if ($timestamp != $task['Task']['next_execution_time']) { + return; + } + if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueFeedCache', $userId, $taskId); + $user = $this->User->getAuthUser($userId); + $this->Job->create(); + $data = array( + 'worker' => 'default', + 'job_type' => 'feed_cache', + 'retries' => 0, + 'org' => $user['Organisation']['name'], + 'org_id' => $user['org_id'], + 'process_id' => 'Part of scheduled feed caching', + 'message' => 'Caching.', + ); + $this->Job->save($data); + $jobId = $this->Job->id; + $result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all'); + $this->Job->save(array( + 'message' => 'Job done.', + 'progress' => 100, + 'status' => 4 + )); + $this->Task->id = $task['Task']['id']; + $this->Task->saveField('message', 'Job completed at ' . date('d/m/Y - H:i:s')); + } public function enqueuePush() { $timestamp = $this->args[0]; diff --git a/app/Controller/TasksController.php b/app/Controller/TasksController.php index b0fbaaf04..715fa40e2 100644 --- a/app/Controller/TasksController.php +++ b/app/Controller/TasksController.php @@ -122,11 +122,17 @@ class TasksController extends AppController { } private function _feedScheduler($timestamp, $id, $type) { + if ($type == 1){ + $action = 'enqueueFeedCache'; + } + else { + $action = 'enqueueFeedFetch'; + } $process_id = CakeResque::enqueueAt( $timestamp, 'default', 'ServerShell', - array('enqueueFeed', $timestamp, $this->Auth->user('id'), $id, $type), + array($action, $timestamp, $this->Auth->user('id'), $id), true ); $this->Task->id = $id; diff --git a/app/Model/Task.php b/app/Model/Task.php index dbc643e4d..4139d0555 100644 --- a/app/Model/Task.php +++ b/app/Model/Task.php @@ -42,8 +42,8 @@ class Task extends AppModel { 'next_execution_time' => 1391601600, 'message' => 'Not scheduled yet.' ), - 'pull_feeds' => array( - 'type' => 'pull_feeds', + 'fetch_feeds' => array( + 'type' => 'fetch_feeds', 'timer' => 0, 'scheduled_time' => '12:00', 'process_id' => '', From 3db62405cd6badc8b540914000d31bd905612669 Mon Sep 17 00:00:00 2001 From: Olivier BERT Date: Fri, 15 Sep 2017 10:35:29 +0200 Subject: [PATCH 17/19] Improved the accessibility of the "Scheduled tasks" page for screen reader. The "aria-label" of the buttons for each tasks (frequency, time, date) should be set to their value rather than their meaning. In fact, the meaning of the value is given by the header of the column, which is already perfectly read by all screen reader I have tested. --- app/View/Tasks/index.ctp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/View/Tasks/index.ctp b/app/View/Tasks/index.ctp index 9a962b04d..277eaaa4b 100644 --- a/app/View/Tasks/index.ctp +++ b/app/View/Tasks/index.ctp @@ -48,7 +48,7 @@ foreach ($list as $item):?> 'id' => $item['Task']['id'] . '-timer-active' )); ?> -
+
@@ -63,7 +63,7 @@ foreach ($list as $item):?> )); ?>
-
+
@@ -78,7 +78,7 @@ foreach ($list as $item):?> )); ?>
-
+
" onClick="activate1(, 'next_execution_time')">
From 700bfb564c428293b4b87a69b08afac013fd57c7 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 15 Sep 2017 15:22:56 +0200 Subject: [PATCH 18/19] Return empty STIX when no data, fixes #2478 --- app/Model/Event.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index acf1ce28b..7c4fb643b 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2885,13 +2885,9 @@ class Event extends AppModel { $eventIDs = $this->Attribute->dissectArgs($id); $tagIDs = $this->Attribute->dissectArgs($tags); $idList = $this->getAccessibleEventIds($eventIDs[0], $eventIDs[1], $tagIDs[0], $tagIDs[1]); - if (empty($idList)) { - return array('success' => 0, 'message' => 'No matching events found to export.'); - } - $event_ids = $this->fetchEventIds($user, $from, $to, $last, true); - $event_ids = array_intersect($event_ids, $idList); - if (empty($event_ids)) { - return array('success' => 0, 'message' => 'No matching events found to export.'); + if (!empty($idList)) { + $event_ids = $this->fetchEventIds($user, $from, $to, $last, true); + $event_ids = array_intersect($event_ids, $idList); } $randomFileName = $this->generateRandomFileName(); $tmpDir = APP . "files" . DS . "scripts" . DS . "tmp"; @@ -2970,12 +2966,9 @@ class Event extends AppModel { } else { $stixFile->append("]}\n"); } - if ($i == 0) { + if($tempFile) { $tempFile->delete(); - $stixFile->delete(); - return array('success' => 0, 'message' => 'No matching events found to export.'); } - $tempFile->delete(); if (!$returnFile) { $data = $stixFile->read(); $stixFile->delete(); From 1f09b445ba71dd2733a5a8ffba46f60eb97bfeb0 Mon Sep 17 00:00:00 2001 From: Olivier BERT Date: Fri, 15 Sep 2017 16:07:33 +0200 Subject: [PATCH 19/19] Accessibility improvement: Given the "button" role and appropriate aria-label to the "Add new cluster" button in the "galaxy quick preview" on an events/view page. --- app/View/Elements/galaxyQuickView.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/galaxyQuickView.ctp b/app/View/Elements/galaxyQuickView.ctp index e9208b565..8557092b5 100644 --- a/app/View/Elements/galaxyQuickView.ctp +++ b/app/View/Elements/galaxyQuickView.ctp @@ -97,7 +97,7 @@ - Add new cluster + Add new cluster