mirror of https://github.com/MISP/MISP
Merge branch '2.4' of github.com:MISP/MISP into pr-5256
mokaddem
commit
dcd3664bba
|
|
@ -40,3 +40,6 @@
|
|||
[submodule "app/files/misp-decaying-models"]
|
||||
path = app/files/misp-decaying-models
|
||||
url = https://github.com/MISP/misp-decaying-models.git
|
||||
[submodule "app/files/scripts/misp-opendata"]
|
||||
path = app/files/scripts/misp-opendata
|
||||
url = https://github.com/MISP/misp-opendata
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,5 +1,5 @@
|
|||
; Generated by RHash v1.3.8 on 2020-05-01 at 12:24.54
|
||||
; Generated by RHash v1.3.9 on 2020-05-17 at 03:12.57
|
||||
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
|
||||
;
|
||||
; 131599 12:24.54 2020-05-01 INSTALL.sh
|
||||
INSTALL.sh 459A7BDAD0014A5BC5BDC4FBB2AEA8A18958D170 EE218F9A83C4E22A6DA65780FB5EB6624104C71A9B3D1743EDB4511AB790B57B B7D0734C17AC66C6B714DCF24CA6053DB31EC635E4AFD2F741601DF2413B1798A9FC9295A8DDE443F315C226DEB9E5A5 D28781C84601334CC88F1C57A869487FAB2CFD199CE83AE658798F3A75777741A49B561C72E85BB9537556E5D3785153339FBD9EEB396088BD0A2DEFE5199319
|
||||
; 132940 03:12.57 2020-05-17 INSTALL.sh
|
||||
INSTALL.sh 89D2EC44902AE611FB65EC5CF30FD8685F713ECD 4D2C48FBF6D668A0FF35B01DE88BAA656943C13A1CEB148C09DD333299E894B3 9BC9C39F584346C8EC1DBF87DBF0E1952EF1736DF22E6BE233A60968F751E2C1C5A8E36CE34BE5696D6EE61D8E1079FA 0AD43002838AF3EB559FC2493A9F9B9FFC1E9592311E0866A46ADA3FB3D0D2AF419B06EB42454184E4DDEDDFAC281C07148E4C61277F933D942ABCEA7CF976CA
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
459a7bdad0014a5bc5bdc4fbb2aea8a18958d170 INSTALL.sh
|
||||
89d2ec44902ae611fb65ec5cf30fd8685f713ecd INSTALL.sh
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
ee218f9a83c4e22a6da65780fb5eb6624104c71a9b3d1743edb4511ab790b57b INSTALL.sh
|
||||
4d2c48fbf6d668a0ff35b01de88baa656943c13a1ceb148c09dd333299e894b3 INSTALL.sh
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
b7d0734c17ac66c6b714dcf24ca6053db31ec635e4afd2f741601df2413b1798a9fc9295a8dde443f315c226deb9e5a5 INSTALL.sh
|
||||
9bc9c39f584346c8ec1dbf87dbf0e1952ef1736df22e6be233a60968f751e2c1c5a8e36ce34be5696d6ee61d8e1079fa INSTALL.sh
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
d28781c84601334cc88f1c57a869487fab2cfd199ce83ae658798f3a75777741a49b561c72e85bb9537556e5d3785153339fbd9eeb396088bd0a2defe5199319 INSTALL.sh
|
||||
0ad43002838af3eb559fc2493a9f9b9ffc1e9592311e0866a46ada3fb3d0d2af419b06eb42454184e4ddeddfac281c07148e4c61277f933d942abcea7cf976ca INSTALL.sh
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
# 0/ Quick MISP Instance on Debian Based Linux - Status |
|
||||
#-------------------------------------------------------|
|
||||
#
|
||||
# 20200513: Ubuntu 20.04 tested and working. -- sCl
|
||||
# 20200412: Ubuntu 18.04.4 tested and working. -- sCl
|
||||
# 20190302: Ubuntu 18.04.2 tested and working. -- sCl
|
||||
# 20190208: Kali Linux tested and working. -- sCl
|
||||
|
|
@ -36,8 +37,9 @@
|
|||
# 2/ For Kali, download and run Installer Script |
|
||||
#-------------------------------------------------------|
|
||||
#
|
||||
# To install MISP on Kali copy paste the following to your r00t shell:
|
||||
# To install MISP on Kali copy paste the following to your shell:
|
||||
# # wget --no-cache -O /tmp/misp-kali.sh https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh && bash /tmp/misp-kali.sh
|
||||
# NO other version then 2020.x supported, kthxbai.
|
||||
# /!\ Please read the installer script before randomly doing the above.
|
||||
# The script is tested on a plain vanilla Kali Linux Boot CD and installs quite a few dependencies.
|
||||
#
|
||||
|
|
@ -115,7 +117,7 @@
|
|||
### END AUTOMATED SECTION ###
|
||||
|
||||
# This function will generate the main installer.
|
||||
# It is a helper function for the maintainers for the installer.
|
||||
# It is a helper function for the maintainers of the installer.
|
||||
|
||||
colors () {
|
||||
# Some colors for easier debug and better UX (not colorblind compatible, PR welcome)
|
||||
|
|
@ -128,13 +130,13 @@ colors () {
|
|||
}
|
||||
|
||||
generateInstaller () {
|
||||
if [ ! -f $(which xsnippet) ]; then
|
||||
if [[ ! -f $(which xsnippet) ]]; then
|
||||
echo 'xsnippet is NOT installed. Clone the repository below and copy the xsnippet shell script somehwere in your $PATH'
|
||||
echo "git clone https://github.com/SteveClement/xsnippet.git"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $(echo $0 |grep -e '^\.\/') != "./INSTALL.tpl.sh" ]]; then
|
||||
if [[ "$(echo $0 |grep -e '^\.\/')" != "./INSTALL.tpl.sh" ]]; then
|
||||
echo -e "${RED}iAmError!${NC}"
|
||||
echo -e "To generate the installer call it with './INSTALL.tpl.sh' otherwise things will break."
|
||||
echo -e "You called: ${RED}$0${NC}"
|
||||
|
|
@ -218,7 +220,7 @@ generateInstaller () {
|
|||
[[ $(type -t debug) == "alias" ]] && unalias debug
|
||||
debug () {
|
||||
echo -e "${RED}Next step:${NC} ${GREEN}$1${NC}" > /dev/tty
|
||||
if [ ! -z $DEBUG ]; then
|
||||
if [[ ! -z ${DEBUG} ]]; then
|
||||
NO_PROGRESS=1
|
||||
echo -e "${RED}Debug Mode${NC}, press ${LBLUE}enter${NC} to continue..." > /dev/tty
|
||||
exec 3>&1
|
||||
|
|
@ -240,7 +242,7 @@ installSupported () {
|
|||
progress 4
|
||||
|
||||
# Check if sudo is installed and etckeeper - functionLocation('generic/sudo_etckeeper.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && checkSudoKeeper 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && checkSudoKeeper
|
||||
[[ ! -z ${MISP_USER} ]] && [[ ! -f /etc/sudoers.d/misp ]] && echo "%${MISP_USER} ALL=(ALL:ALL) NOPASSWD:ALL" |sudo tee /etc/sudoers.d/misp
|
||||
progress 4
|
||||
|
||||
|
|
@ -248,7 +250,7 @@ installSupported () {
|
|||
checkLocale
|
||||
|
||||
# Upgrade system to make sure we install the latest packages - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && aptUpgrade 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && aptUpgrade
|
||||
progress 4
|
||||
|
||||
# TODO: Double check how the user is added and subsequently used during the install.
|
||||
|
|
@ -306,40 +308,40 @@ installSupported () {
|
|||
progress 4
|
||||
|
||||
# Make sure permissions are sane - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && permissions 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && permissions
|
||||
progress 4
|
||||
|
||||
# TODO: Mysql install functions, make it upgrade safe, double check
|
||||
# Setup Databse - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && prepareDB 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && prepareDB
|
||||
progress 4
|
||||
|
||||
# Roll Apache Config - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && apacheConfig 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && apacheConfig
|
||||
progress 4
|
||||
|
||||
# Setup log logrotate - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && logRotation 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && logRotation
|
||||
progress 4
|
||||
|
||||
# Generate MISP Config files - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && configMISP 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && configMISP
|
||||
progress 4
|
||||
|
||||
# Generate GnuPG key - functionLocation('generic/gnupg.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && setupGnuPG 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && setupGnuPG
|
||||
progress 4
|
||||
|
||||
# Setup and start background workers - functionLocation('INSTALL.ubuntu1804.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && backgroundWorkers 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && backgroundWorkers
|
||||
progress 4
|
||||
|
||||
# Run cake CLI for the core installation - functionLocation('generic/MISP_CAKE_init.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && coreCAKE 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && coreCAKE
|
||||
progress 4
|
||||
|
||||
# Update Galaxies, Template Objects, Warning Lists, Notice Lists, Taxonomies - functionLocation('generic/MISP_CAKE_init.md')
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && updateGOWNT 2> /dev/null > /dev/null
|
||||
[[ -n $CORE ]] || [[ -n $ALL ]] && updateGOWNT
|
||||
progress 4
|
||||
|
||||
# Disable spinner
|
||||
|
|
@ -370,7 +372,7 @@ installSupported () {
|
|||
|
||||
# Install misp-dashboard - functionLocation('generic/misp-dashboard-debian.md')
|
||||
## FIXME: The current state of misp-dashboard is broken, disabling any use.
|
||||
##[[ -n $DASHBOARD ]] || [[ -n $ALL ]] && mispDashboard ; dashboardCAKE 2> /dev/null > /dev/null
|
||||
##[[ -n $DASHBOARD ]] || [[ -n $ALL ]] && mispDashboard ; dashboardCAKE
|
||||
##progress 4
|
||||
|
||||
# Install Mail2MISP - functionLocation('generic/mail_to_misp-debian.md')
|
||||
|
|
@ -388,7 +390,7 @@ installSupported () {
|
|||
# Main Kali Install function
|
||||
installMISPonKali () {
|
||||
# Kali might have a bug on installs where libc6 is not up to date, this forces bash and libc to update - functionLocation('')
|
||||
kaliUpgrade 2> /dev/null > /dev/null
|
||||
kaliUpgrade
|
||||
|
||||
# Set locale if not set - functionLocation('generic/supportFunctions.md')
|
||||
checkLocale
|
||||
|
|
@ -397,13 +399,13 @@ installMISPonKali () {
|
|||
setBaseURL
|
||||
|
||||
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
|
||||
installDepsPhp73 2> /dev/null > /dev/null
|
||||
installDepsPhp73
|
||||
|
||||
# Set custom Kali only variables and tweaks
|
||||
space
|
||||
# The following disables sleep on kali/gnome
|
||||
### FIXME: Disabling for now, maybe source of some issues.
|
||||
##disableSleep 2> /dev/null > /dev/null
|
||||
##disableSleep
|
||||
##debug "Sleeping 3 seconds to make sure the disable sleep does not confuse the execution of the script."
|
||||
##sleep 3
|
||||
|
||||
|
|
@ -415,51 +417,50 @@ installMISPonKali () {
|
|||
installCoreDeps
|
||||
|
||||
debug "Enabling redis and gnupg modules"
|
||||
phpenmod -v 7.3 redis
|
||||
phpenmod -v 7.3 gnupg
|
||||
sudo phpenmod -v 7.3 redis
|
||||
sudo phpenmod -v 7.3 gnupg
|
||||
|
||||
debug "Apache2 ops: dismod: status php7.2 - dissite: 000-default enmod: ssl rewrite headers php7.3 ensite: default-ssl"
|
||||
a2dismod status 2> /dev/null > /dev/null
|
||||
a2dismod php7.2 2> /dev/null > /dev/null
|
||||
a2enmod ssl rewrite headers php7.3 2> /dev/null > /dev/null
|
||||
a2dissite 000-default 2> /dev/null > /dev/null
|
||||
a2ensite default-ssl 2> /dev/null > /dev/null
|
||||
debug "Apache2 ops: dismod: status - dissite: 000-default enmod: ssl rewrite headers php7.3 ensite: default-ssl"
|
||||
sudo a2dismod status
|
||||
sudo a2enmod ssl rewrite headers php7.3
|
||||
sudo a2dissite 000-default
|
||||
sudo a2ensite default-ssl
|
||||
|
||||
debug "Restarting mysql.service"
|
||||
systemctl restart mysql.service 2> /dev/null > /dev/null
|
||||
sudo systemctl restart mysql.service
|
||||
|
||||
debug "Fixing redis rc script on Kali"
|
||||
fixRedis 2> /dev/null > /dev/null
|
||||
fixRedis
|
||||
|
||||
debug "git clone, submodule update everything"
|
||||
mkdir $PATH_TO_MISP
|
||||
chown $WWW_USER:$WWW_USER $PATH_TO_MISP
|
||||
sudo mkdir $PATH_TO_MISP
|
||||
sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP
|
||||
cd $PATH_TO_MISP
|
||||
$SUDO_WWW git clone https://github.com/MISP/MISP.git $PATH_TO_MISP
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MISP/MISP.git $PATH_TO_MISP; done
|
||||
|
||||
$SUDO_WWW git config core.filemode false
|
||||
|
||||
cd $PATH_TO_MISP
|
||||
$SUDO_WWW git submodule update --init --recursive 2> /dev/null > /dev/null
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git submodule update --progress --init --recursive; done
|
||||
# Make git ignore filesystem permission differences for submodules
|
||||
$SUDO_WWW git submodule foreach --recursive git config core.filemode false
|
||||
|
||||
cd $PATH_TO_MISP/app/files/scripts
|
||||
$SUDO_WWW git clone https://github.com/CybOXProject/python-cybox.git 2> /dev/null > /dev/null
|
||||
$SUDO_WWW git clone https://github.com/STIXProject/python-stix.git 2> /dev/null > /dev/null
|
||||
$SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git 2> /dev/null > /dev/null
|
||||
$SUDO_WWW git clone https://github.com/MAECProject/python-maec.git 2> /dev/null > /dev/null
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/CybOXProject/python-cybox.git; done
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/STIXProject/python-stix.git; done
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git; done
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MAECProject/python-maec.git; done
|
||||
|
||||
|
||||
mkdir /var/www/.cache/
|
||||
sudo mkdir /var/www/.cache/
|
||||
|
||||
MISP_USER_HOME=$(sudo -Hiu $MISP_USER env | grep HOME |cut -f 2 -d=)
|
||||
mkdir $MISP_USER_HOME/.cache
|
||||
chown $MISP_USER:$MISP_USER $MISP_USER_HOME/.cache
|
||||
chown $WWW_USER:$WWW_USER /var/www/.cache
|
||||
sudo mkdir $MISP_USER_HOME/.cache
|
||||
sudo chown $MISP_USER:$MISP_USER $MISP_USER_HOME/.cache
|
||||
sudo chown $WWW_USER:$WWW_USER /var/www/.cache
|
||||
|
||||
debug "Generating rc.local"
|
||||
genRCLOCAL
|
||||
## Not really needed...
|
||||
## debug "Generating rc.local"
|
||||
## genRCLOCAL
|
||||
|
||||
debug "Setting up main MISP virtualenv"
|
||||
# Needs virtualenv
|
||||
|
|
@ -471,69 +472,62 @@ installMISPonKali () {
|
|||
|
||||
debug "Installing python-cybox"
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-cybox
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install . 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
||||
debug "Installing python-stix"
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-stix
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install . 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
||||
debug "Install maec"
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-maec
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install . 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
||||
# install STIX2.0 library to support STIX 2.0 export
|
||||
debug "Installing cti-python-stix2"
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I antlr4-python3-runtime==4.7.2 2> /dev/null > /dev/null
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd ${PATH_TO_MISP}/cti-python-stix2
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
||||
debug "Installing mixbox"
|
||||
cd $PATH_TO_MISP/app/files/scripts/mixbox
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install . 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
||||
# install PyMISP
|
||||
debug "Installing PyMISP"
|
||||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install . 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
||||
# install pydeep
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git 2> /dev/null > /dev/null
|
||||
false; while [[ $? -ne 0 ]]; do $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git; done
|
||||
|
||||
# install lief
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install lief 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install lief
|
||||
|
||||
# install python-magic
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic
|
||||
|
||||
# install plyara
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara
|
||||
|
||||
# install zmq needed by mispzmq
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq 2> /dev/null > /dev/null
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq
|
||||
|
||||
# Install Crypt_GPG and Console_CommandLine
|
||||
debug "Installing pear Console_CommandLine"
|
||||
pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml
|
||||
debug "Installing pear Crypt_GPG"
|
||||
pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml
|
||||
|
||||
|
||||
debug "Installing composer with php 7.3 updates"
|
||||
composer73
|
||||
debug "Installing cake"
|
||||
composer
|
||||
|
||||
$SUDO_WWW cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php
|
||||
|
||||
chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP
|
||||
chmod -R 750 $PATH_TO_MISP
|
||||
chmod -R g+ws $PATH_TO_MISP/app/tmp
|
||||
chmod -R g+ws $PATH_TO_MISP/app/files
|
||||
chmod -R g+ws $PATH_TO_MISP/app/files/scripts/tmp
|
||||
sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP
|
||||
sudo chmod -R 750 $PATH_TO_MISP
|
||||
sudo chmod -R g+ws $PATH_TO_MISP/app/tmp
|
||||
sudo chmod -R g+ws $PATH_TO_MISP/app/files
|
||||
sudo chmod -R g+ws $PATH_TO_MISP/app/files/scripts/tmp
|
||||
|
||||
debug "Setting up database"
|
||||
if [[ ! -e /var/lib/mysql/misp/users.ibd ]]; then
|
||||
echo "
|
||||
set timeout 10
|
||||
spawn mysql_secure_installation
|
||||
spawn sudo mysql_secure_installation
|
||||
expect \"Enter current password for root (enter for none):\"
|
||||
send -- \"\r\"
|
||||
expect \"Set root password?\"
|
||||
|
|
@ -552,13 +546,14 @@ installMISPonKali () {
|
|||
send -- \"y\r\"
|
||||
expect eof" | expect -f -
|
||||
|
||||
mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;"
|
||||
mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE ON *.* TO $DBUSER_MISP@localhost IDENTIFIED BY '$DBPASSWORD_MISP';"
|
||||
mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT ALL PRIVILEGES ON $DBNAME.* TO '$DBUSER_MISP'@'localhost';"
|
||||
mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "FLUSH PRIVILEGES;"
|
||||
sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;"
|
||||
sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE ON *.* TO $DBUSER_MISP@localhost IDENTIFIED BY '$DBPASSWORD_MISP';"
|
||||
sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT ALL PRIVILEGES ON $DBNAME.* TO '$DBUSER_MISP'@'localhost';"
|
||||
sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "FLUSH PRIVILEGES;"
|
||||
|
||||
enableServices
|
||||
|
||||
debug "Populating database"
|
||||
$SUDO_WWW cat $PATH_TO_MISP/INSTALL/MYSQL.sql | mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME
|
||||
|
||||
echo "<?php
|
||||
|
|
@ -576,7 +571,7 @@ installMISPonKali () {
|
|||
'prefix' => '',
|
||||
'encoding' => 'utf8',
|
||||
);
|
||||
}" | $SUDO_WWW tee $PATH_TO_MISP/app/Config/database.php 2> /dev/null > /dev/null
|
||||
}" | $SUDO_WWW tee $PATH_TO_MISP/app/Config/database.php
|
||||
else
|
||||
echo "There might be a database already existing here: /var/lib/mysql/misp/users.ibd"
|
||||
echo "Skipping any creations…"
|
||||
|
|
@ -584,56 +579,55 @@ installMISPonKali () {
|
|||
fi
|
||||
|
||||
debug "Generating Certificate"
|
||||
openssl req -newkey rsa:4096 -days 365 -nodes -x509 \
|
||||
sudo openssl req -newkey rsa:4096 -days 365 -nodes -x509 \
|
||||
-subj "/C=${OPENSSL_C}/ST=${OPENSSL_ST}/L=${OPENSSL_L}/O=${OPENSSL_O}/OU=${OPENSSL_OU}/CN=${OPENSSL_CN}/emailAddress=${OPENSSL_EMAILADDRESS}" \
|
||||
-keyout /etc/ssl/private/misp.local.key -out /etc/ssl/private/misp.local.crt
|
||||
|
||||
debug "Generating Apache Conf"
|
||||
genApacheConf
|
||||
|
||||
echo "127.0.0.1 misp.local" | tee -a /etc/hosts
|
||||
echo "127.0.0.1 misp.local" | sudo tee -a /etc/hosts
|
||||
|
||||
debug "Disabling site default-ssl, enabling misp-ssl"
|
||||
a2dissite default-ssl
|
||||
a2ensite misp-ssl
|
||||
sudo a2dissite default-ssl
|
||||
sudo a2ensite misp-ssl
|
||||
|
||||
for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit
|
||||
do
|
||||
sed -i "s/^\($key\).*/\1 = $(eval echo \${$key})/" $PHP_INI
|
||||
sudo sed -i "s/^\($key\).*/\1 = $(eval echo \${$key})/" $PHP_INI
|
||||
done
|
||||
|
||||
debug "Restarting Apache2"
|
||||
systemctl restart apache2
|
||||
sudo systemctl restart apache2
|
||||
|
||||
debug "Setting up logrotate"
|
||||
cp $PATH_TO_MISP/INSTALL/misp.logrotate /etc/logrotate.d/misp
|
||||
chmod 0640 /etc/logrotate.d/misp
|
||||
sudo cp $PATH_TO_MISP/INSTALL/misp.logrotate /etc/logrotate.d/misp
|
||||
sudo chmod 0640 /etc/logrotate.d/misp
|
||||
|
||||
$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/bootstrap.default.php $PATH_TO_MISP/app/Config/bootstrap.php
|
||||
$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/core.default.php $PATH_TO_MISP/app/Config/core.php
|
||||
$SUDO_WWW cp -a $PATH_TO_MISP/app/Config/config.default.php $PATH_TO_MISP/app/Config/config.php
|
||||
|
||||
chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/app/Config
|
||||
chmod -R 750 $PATH_TO_MISP/app/Config
|
||||
sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/app/Config
|
||||
sudo chmod -R 750 $PATH_TO_MISP/app/Config
|
||||
|
||||
debug "Setting up GnuPG"
|
||||
setupGnuPG 2> /dev/null > /dev/null
|
||||
setupGnuPG
|
||||
|
||||
debug "Adding workers to systemd"
|
||||
chmod +x $PATH_TO_MISP/app/Console/worker/start.sh
|
||||
sudo cp $PATH_TO_MISP/INSTALL/misp-workers.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now misp-workers
|
||||
backgroundWorkers
|
||||
|
||||
debug "Running Core Cake commands"
|
||||
coreCAKE 2> /dev/null > /dev/null
|
||||
coreCAKE
|
||||
|
||||
## FIXME: The current state of misp-dashboard is broken, disabling any use.
|
||||
##dashboardCAKE 2> /dev/null > /dev/null
|
||||
##dashboardCAKE
|
||||
|
||||
debug "Update: Galaxies, Template Objects, Warning Lists, Notice Lists, Taxonomies"
|
||||
updateGOWNT 2> /dev/null > /dev/null
|
||||
updateGOWNT
|
||||
|
||||
gitPullAllRCLOCAL
|
||||
# This is not needed atm...
|
||||
##gitPullAllRCLOCAL
|
||||
|
||||
checkUsrLocalSrc
|
||||
|
||||
|
|
@ -646,7 +640,7 @@ installMISPonKali () {
|
|||
|
||||
debug "Installing ssdeep"
|
||||
ssdeep
|
||||
phpenmod -v 7.3 ssdeep
|
||||
sudo phpenmod -v 7.3 ssdeep
|
||||
|
||||
debug "Setting permissions"
|
||||
permissions
|
||||
|
|
@ -665,17 +659,17 @@ installMISPRHEL () {
|
|||
|
||||
if [[ -n $CORE ]] || [[ -n $ALL ]]; then
|
||||
space
|
||||
echo "Proceeding with MISP core installation on RHEL $dist_version"
|
||||
echo "Proceeding with MISP core installation on RHEL ${dist_version}"
|
||||
space
|
||||
|
||||
id -u "$MISP_USER" > /dev/null
|
||||
if [ $? -eq 1 ]; then
|
||||
id -u "${MISP_USER}" > /dev/null
|
||||
if [[ $? -eq 1 ]]; then
|
||||
debug "Creating MISP user"
|
||||
sudo useradd -r "$MISP_USER"
|
||||
sudo useradd -r "${MISP_USER}"
|
||||
fi
|
||||
|
||||
debug "Enabling Extras Repos (SCL)"
|
||||
if [[ $FLAVOUR == "rhel" ]]; then
|
||||
if [[ "${FLAVOUR}" == "rhel" ]]; then
|
||||
sudo subscription-manager register --auto-attach
|
||||
enableReposRHEL
|
||||
enableEPEL
|
||||
|
|
@ -756,7 +750,7 @@ debug "Setting MISP variables"
|
|||
MISPvars
|
||||
|
||||
debug "Checking for parameters or Unattended Kali Install"
|
||||
if [[ $# == 0 && $0 != "/tmp/misp-kali.sh" ]]; then
|
||||
if [[ $# -eq 0 && "$0" != "/tmp/misp-kali.sh" ]]; then
|
||||
usage
|
||||
exit
|
||||
else
|
||||
|
|
@ -801,10 +795,6 @@ x86_64-debian-stretch
|
|||
x86_64-debian-buster
|
||||
x86_64-ubuntu-bionic
|
||||
x86_64-ubuntu-focal
|
||||
x86_64-kali-2019.1
|
||||
x86_64-kali-2019.2
|
||||
x86_64-kali-2019.3
|
||||
x86_64-kali-2019.4
|
||||
x86_64-kali-2020.1
|
||||
x86_64-kali-2020.2
|
||||
x86_64-kali-2020.3
|
||||
|
|
@ -830,30 +820,30 @@ EOF
|
|||
fi
|
||||
|
||||
# If Ubuntu is detected, figure out which release it is and run the according scripts
|
||||
if [ "${FLAVOUR}" == "ubuntu" ]; then
|
||||
if [[ "${FLAVOUR}" == "ubuntu" ]]; then
|
||||
RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]')
|
||||
if [ "${RELEASE}" == "18.04" ]; then
|
||||
if [[ "${RELEASE}" == "18.04" ]]; then
|
||||
echo "Install on Ubuntu 18.04 LTS fully supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported && exit || exit
|
||||
fi
|
||||
if [ "${RELEASE}" == "20.04" ]; then
|
||||
if [[ "${RELEASE}" == "20.04" ]]; then
|
||||
echo "Install on Ubuntu 20.04 LTS fully supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported PHP="7.4" && exit || exit
|
||||
fi
|
||||
if [ "${RELEASE}" == "18.10" ]; then
|
||||
if [[ "${RELEASE}" == "18.10" ]]; then
|
||||
echo "Install on Ubuntu 18.10 partially supported, bye."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported && exit || exit
|
||||
fi
|
||||
if [ "${RELEASE}" == "19.04" ]; then
|
||||
if [[ "${RELEASE}" == "19.04" ]]; then
|
||||
echo "Install on Ubuntu 19.04 partially supported bye."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported && exit || exit
|
||||
exit 1
|
||||
fi
|
||||
if [ "${RELEASE}" == "19.10" ]; then
|
||||
if [[ "${RELEASE}" == "19.10" ]]; then
|
||||
echo "Install on Ubuntu 19.10 not supported, bye"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -862,19 +852,19 @@ if [ "${FLAVOUR}" == "ubuntu" ]; then
|
|||
fi
|
||||
|
||||
# If Debian is detected, figure out which release it is and run the according scripts
|
||||
if [ "${FLAVOUR}" == "debian" ]; then
|
||||
if [[ "${FLAVOUR}" == "debian" ]]; then
|
||||
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
|
||||
if [ "${CODE}" == "buster" ]; then
|
||||
if [[ "${CODE}" == "buster" ]]; then
|
||||
echo "Install on Debian testing fully supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported PHP=7.3 && exit || exit
|
||||
fi
|
||||
if [ "${CODE}" == "sid" ]; then
|
||||
if [[ "${CODE}" == "sid" ]]; then
|
||||
echo "Install on Debian unstable not fully supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported PHP=7.3 && exit || exit
|
||||
fi
|
||||
if [ "${CODE}" == "stretch" ]; then
|
||||
if [[ "${CODE}" == "stretch" ]]; then
|
||||
echo "Install on Debian stable fully supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
installSupported PHP=7.0 && exit || exit
|
||||
|
|
@ -884,13 +874,13 @@ if [ "${FLAVOUR}" == "debian" ]; then
|
|||
fi
|
||||
|
||||
# If Tsurugi is detected, figure out which release it is and run the according scripts
|
||||
if [ "${FLAVOUR}" == "tsurugi" ]; then
|
||||
if [[ "${FLAVOUR}" == "tsurugi" ]]; then
|
||||
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
|
||||
if [ "${CODE}" == "bamboo" ]; then
|
||||
if [[ "${CODE}" == "bamboo" ]]; then
|
||||
echo "Install on Tsurugi Lab partially supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
fi
|
||||
if [ "${CODE}" == "soy sauce" ]; then
|
||||
if [[ "${CODE}" == "soy sauce" ]]; then
|
||||
echo "Install on Tsurugi Acquire partially supported."
|
||||
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
|
||||
fi
|
||||
|
|
@ -899,16 +889,16 @@ if [ "${FLAVOUR}" == "tsurugi" ]; then
|
|||
fi
|
||||
|
||||
# If Kali Linux is detected, run the acccording scripts
|
||||
if [ "${FLAVOUR}" == "kali" ]; then
|
||||
if [[ "${FLAVOUR}" == "kali" ]]; then
|
||||
KALI=1
|
||||
kaliOnRootR0ckz
|
||||
kaliOnTheR0ckz
|
||||
installMISPonKali
|
||||
echo "Installation done!"
|
||||
exit
|
||||
fi
|
||||
|
||||
# If RHEL/CentOS is detected, run appropriate script
|
||||
if [ "${FLAVOUR}" == "rhel" ] || [ "${FLAVOUR}" == "centos" ]; then
|
||||
if [[ "${FLAVOUR}" == "rhel" ]] || [[ "${FLAVOUR}" == "centos" ]]; then
|
||||
installMISPRHEL
|
||||
echo "Installation done !"
|
||||
exit
|
||||
|
|
|
|||
2
PyMISP
2
PyMISP
|
|
@ -1 +1 @@
|
|||
Subproject commit c098981a40b72d0eb277be5c991e5949adee2fa5
|
||||
Subproject commit 1bb1ae16041b43fd7f3995c1417d4aba206bafed
|
||||
|
|
@ -1 +1 @@
|
|||
{"major":2, "minor":4, "hotfix":125}
|
||||
{"major":2, "minor":4, "hotfix":126}
|
||||
|
|
|
|||
|
|
@ -25,16 +25,8 @@ class AuthkeyShell extends AppShell {
|
|||
$this->User->id = $user['User']['id'];
|
||||
$newkey = $this->User->generateAuthKey();
|
||||
if ($this->User->saveField('authkey', $newkey)) {
|
||||
$this->Log->create();
|
||||
$this->Log->save(array(
|
||||
'org' => $user['Organisation']['name'],
|
||||
'model' => 'User',
|
||||
'model_id' => $user['User']['id'],
|
||||
'email' => 'SYSTEM',
|
||||
'action' => 'reset_auth_key',
|
||||
'title' => 'Authentication key for user ' . $user['User']['id'] . ' (' . $user['User']['email'] . ')',
|
||||
'change' => 'authkey(' . $user['User']['authkey'] . ') => (' . $newkey . ')'
|
||||
));
|
||||
$logTitle = 'Authentication key for user ' . $user['User']['id'] . ' (' . $user['User']['email'] . ')';
|
||||
$this->Log->createLogEntry('SYSTEM', 'reset_auth_key', 'User', $user['User']['id'], $logTitle, array('authkey' => array($user['User']['authkey'], $newkey)));
|
||||
echo $newkey . PHP_EOL;
|
||||
} else {
|
||||
echo 'Could not update account for User.id = ', $user['User']['id'], PHP_EOL;
|
||||
|
|
|
|||
|
|
@ -70,9 +70,13 @@ class ServerShell extends AppShell
|
|||
$this->Job->save($data);
|
||||
$jobId = $this->Job->id;
|
||||
}
|
||||
$force = false;
|
||||
if (!empty($this->args[4]) && $this->args[4] === 'force') {
|
||||
$force = true;
|
||||
}
|
||||
$this->Server->id = $serverId;
|
||||
$server = $this->Server->read(null, $serverId);
|
||||
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId);
|
||||
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
|
||||
$this->Job->id = $jobId;
|
||||
$this->Job->save(array(
|
||||
'id' => $jobId,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class AppController extends Controller
|
|||
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
|
||||
|
||||
private $__queryVersion = '106';
|
||||
public $pyMispVersion = '2.4.125';
|
||||
public $pyMispVersion = '2.4.126';
|
||||
public $phpmin = '7.2';
|
||||
public $phprec = '7.4';
|
||||
public $pythonmin = '3.6';
|
||||
|
|
|
|||
|
|
@ -3148,7 +3148,7 @@ class AttributesController extends AppController
|
|||
|
||||
public function toggleCorrelation($id)
|
||||
{
|
||||
if (!$this->_isSiteAdmin() && Configure::read('MISP.allow_disabling_correlation')) {
|
||||
if (!$this->_isSiteAdmin() && !Configure::read('MISP.allow_disabling_correlation')) {
|
||||
throw new MethodNotAllowedException(__('Disabling the correlation is not permitted on this instance.'));
|
||||
}
|
||||
$this->Attribute->id = $id;
|
||||
|
|
|
|||
|
|
@ -5244,7 +5244,7 @@ class EventsController extends AppController
|
|||
|
||||
public function toggleCorrelation($id)
|
||||
{
|
||||
if (!$this->_isSiteAdmin() && Configure.read('MISP.allow_disabling_correlation')) {
|
||||
if (!$this->_isSiteAdmin() && !Configure::read('MISP.allow_disabling_correlation')) {
|
||||
throw new MethodNotAllowedException(__('Disabling the correlation is not permitted on this instance.'));
|
||||
}
|
||||
$this->Event->id = $id;
|
||||
|
|
|
|||
|
|
@ -364,7 +364,11 @@ class FeedsController extends AppController
|
|||
}
|
||||
}
|
||||
if (!isset($this->request->data['Feed']['settings'])) {
|
||||
$this->request->data['Feed']['settings'] = array();
|
||||
if (!empty($this->Feed->data['Feed']['settings'])) {
|
||||
$this->request->data['Feed']['settings'] = $this->Feed->data['Feed']['settings'];
|
||||
} else {
|
||||
$this->request->data['Feed']['settings'] = array();
|
||||
}
|
||||
} else {
|
||||
if (!empty($this->request->data['Feed']['settings']['common']['excluderegex']) && !$this->__checkRegex($this->request->data['Feed']['settings']['common']['excluderegex'])) {
|
||||
$this->Flash->error('Invalid exclude regex. Make sure it\'s a delimited PCRE regex pattern.');
|
||||
|
|
|
|||
|
|
@ -1624,6 +1624,7 @@ class ServersController extends AppController
|
|||
$result = $pubSubTool->statusCheck();
|
||||
if (!empty($result)) {
|
||||
$this->set('events', $result['publishCount']);
|
||||
$this->set('messages', $result['messageCount']);
|
||||
$this->set('time', date('Y/m/d H:i:s', $result['timestamp']));
|
||||
$this->set('time2', date('Y/m/d H:i:s', $result['timestampSettings']));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
<?php
|
||||
|
||||
class OpendataExport
|
||||
{
|
||||
public $non_restrictive_export = true;
|
||||
public $use_default_filters = true;
|
||||
public $mock_query_only = true;
|
||||
private $__default_filters = null;
|
||||
|
||||
private $__auth = null;
|
||||
private $__delete = false;
|
||||
private $__scope = null;
|
||||
private $__setup = array();
|
||||
private $__url = null;
|
||||
|
||||
private $__scripts_dir = APP . 'files/scripts/';
|
||||
private $__script_name = 'misp-opendata/opendata.py';
|
||||
|
||||
public function setDefaultFilters($filters)
|
||||
{
|
||||
$this->__default_filters = $filters;
|
||||
}
|
||||
|
||||
public function header($options = array())
|
||||
{
|
||||
$this->__scope = $options['scope'];
|
||||
if (isset($this->__default_filters['auth'])) {
|
||||
$this->__auth = $this->__default_filters['auth'];
|
||||
unset($this->__default_filters['auth']);
|
||||
}
|
||||
if (isset($this->__default_filters['setup'])) {
|
||||
$this->__setup = $this->__default_filters['setup'];
|
||||
$this->__check_setup_filter();
|
||||
unset($this->__default_filters['setup']);
|
||||
} else {
|
||||
throw new Exception(__('Missing "setup" filter containing the dataset and resource(s) information.'));
|
||||
}
|
||||
if (isset($this->__default_filters['url'])) {
|
||||
$this->__url = $this->__default_filters['url'];
|
||||
unset($this->__default_filters['url']);
|
||||
} else {
|
||||
$external_baseurl = Configure::read('MISP.external_baseurl');
|
||||
$baseurl = !empty($external_baseurl) ? $external_baseurl : Configure::read('MISP.baseurl');
|
||||
if (empty($baseurl)) {
|
||||
throw new Exception(__('Missing url of the MISP instance, and baseurl is not set.'));
|
||||
}
|
||||
$this->__url = $baseurl;
|
||||
}
|
||||
if (!empty($this->__default_filters['delete'])) {
|
||||
$this->__delete = true;
|
||||
unset($this->__default_filters['delete']);
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
public function footer()
|
||||
{
|
||||
$authParam = ' --auth ' . $this->__auth;
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$cmd = $my_server->getPythonVersion() . ' ' . $this->__scripts_dir . $this->__script_name . $authParam;
|
||||
return $this->__delete ? $this->__delete_query($cmd) : $this->__add_query($cmd);
|
||||
}
|
||||
|
||||
public function separator()
|
||||
{
|
||||
return '';
|
||||
}
|
||||
|
||||
private function __add_query($cmd)
|
||||
{
|
||||
unset($this->__default_filters['returnFormat']);
|
||||
$body = json_encode($this->__default_filters);
|
||||
$bodyFilename = $this->__generateSetupFile($body);
|
||||
$bodyParam = ' --body ' . $bodyFilename;
|
||||
$levelParam = ' --level ' . strtolower($this->__scope) . 's';
|
||||
$setup = json_encode($this->__setup);
|
||||
$setupFilename = $this->__generateSetupFile($setup);
|
||||
$setupParam = ' --setup ' . $setupFilename;
|
||||
$urlParam = ' --url ' . $this->__url;
|
||||
|
||||
$cmd .= $bodyParam . $setupParam . $levelParam . $urlParam;
|
||||
$results = shell_exec($cmd);
|
||||
unlink($bodyFilename);
|
||||
unlink($setupFilename);
|
||||
return $results;
|
||||
}
|
||||
|
||||
private function __check_setup_filter()
|
||||
{
|
||||
if (empty($this->__setup['dataset'])) {
|
||||
throw new Exception(__('Missing dataset filter in the setup filter. Please provide the dataset setup.'));
|
||||
}
|
||||
if (!empty($this->__setup['resources']) && !empty($this->__setup['resource'])) {
|
||||
throw new Exception(__('Please provide the resource setup in a single field called "resources".'));
|
||||
}
|
||||
if (!empty($this->__setup['resource']) && empty($this->__setup['resources'])) {
|
||||
$this->__setup['resources'] = $this->__setup['resource'];
|
||||
unset($this->__setup['resource']);
|
||||
}
|
||||
}
|
||||
|
||||
private function __delete_query($cmd)
|
||||
{
|
||||
$cmd .= " -d '" . $this->__setup['dataset'] . "'";
|
||||
if (!empty($this->__setup['resources'])) {
|
||||
if (is_array($this->__setup['resources'])) {
|
||||
foreach ($this->__setup['resources'] as $resource) {
|
||||
$cmd .= ' ' . $resource;
|
||||
}
|
||||
} else {
|
||||
$cmd .= " '" . $this->__setup['resources'] . "'";
|
||||
}
|
||||
}
|
||||
return shell_exec($cmd);
|
||||
}
|
||||
|
||||
private function __generateRandomFileName()
|
||||
{
|
||||
return (new RandomTool())->random_str(false, 12);
|
||||
}
|
||||
|
||||
private function __generateSetupFile($to_write)
|
||||
{
|
||||
$filename = $this->__scripts_dir . 'tmp/' . $this->__generateRandomFileName();
|
||||
$tmpFile = new File($filename, true, 0644);
|
||||
$tmpFile->write($to_write);
|
||||
$tmpFile->close();
|
||||
return $filename;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,66 +1,53 @@
|
|||
<?php
|
||||
class PubSubTool
|
||||
{
|
||||
private $__redis = false;
|
||||
private $__settings = false;
|
||||
const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
|
||||
const OLD_PID_LOCATION = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid';
|
||||
|
||||
private function __getSetSettings()
|
||||
{
|
||||
$settings = array(
|
||||
'redis_host' => 'localhost',
|
||||
'redis_port' => '6379',
|
||||
'redis_password' => '',
|
||||
'redis_database' => '1',
|
||||
'redis_namespace' => 'mispq',
|
||||
'port' => '50000',
|
||||
);
|
||||
/**
|
||||
* @var Redis
|
||||
*/
|
||||
private $redis;
|
||||
|
||||
foreach ($settings as $key => $setting) {
|
||||
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
|
||||
if ($temp) {
|
||||
$settings[$key] = $temp;
|
||||
}
|
||||
}
|
||||
$settingsFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'settings.json', true, 0644);
|
||||
$settingsFile->write(json_encode($settings, true));
|
||||
$settingsFile->close();
|
||||
return $settings;
|
||||
}
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
private $settings;
|
||||
|
||||
public function initTool()
|
||||
{
|
||||
if (!$this->__redis) {
|
||||
$settings = $this->__setupPubServer();
|
||||
$redis = new Redis();
|
||||
$redis->connect($settings['redis_host'], $settings['redis_port']);
|
||||
$redis_pwd = $settings['redis_password'];
|
||||
if (!empty($redis_pwd)) {
|
||||
$redis->auth($redis_pwd);
|
||||
}
|
||||
$redis->select($settings['redis_database']);
|
||||
$this->__redis = $redis;
|
||||
$this->__settings = $settings;
|
||||
} else {
|
||||
$settings = $this->__settings;
|
||||
if (!$this->redis) {
|
||||
$settings = $this->getSetSettings();
|
||||
$this->setupPubServer($settings);
|
||||
$this->redis = $this->createRedisConnection($settings);
|
||||
$this->settings = $settings;
|
||||
}
|
||||
return $settings;
|
||||
}
|
||||
|
||||
// read the pid file, if it exists, check if the process is actually running
|
||||
// if either the pid file doesn't exists or the process is not running return false
|
||||
// otherwise return the pid
|
||||
public function checkIfRunning()
|
||||
/**
|
||||
* Read the pid file, if it exists, check if the process is actually running
|
||||
* if either the pid file doesn't exists or the process is not running return false
|
||||
* otherwise return the pid.
|
||||
*
|
||||
* @param string|null $pidFilePath
|
||||
* @return bool|int False when process is not running, PID otherwise.
|
||||
* @throws Exception
|
||||
*/
|
||||
public function checkIfRunning($pidFilePath = null)
|
||||
{
|
||||
$pidFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid');
|
||||
$pid = $pidFile->read(true, 'r');
|
||||
$pidFile = new File($pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid');
|
||||
if (!$pidFile->exists()) {
|
||||
return false;
|
||||
}
|
||||
$pid = $pidFile->read();
|
||||
if ($pid === false || $pid === '') {
|
||||
return false;
|
||||
}
|
||||
if (!is_numeric($pid)) {
|
||||
throw new Exception('Internal error (invalid PID file for the MISP zmq script)');
|
||||
}
|
||||
$result = trim(shell_exec('ps aux | awk \'{print $2}\' | grep "^' . $pid . '$"'));
|
||||
if (empty($result)) {
|
||||
$result = file_exists("/proc/$pid");
|
||||
if ($result === false) {
|
||||
return false;
|
||||
}
|
||||
return $pid;
|
||||
|
|
@ -68,18 +55,14 @@ class PubSubTool
|
|||
|
||||
public function statusCheck()
|
||||
{
|
||||
$redis = new Redis();
|
||||
$settings = $this->__getSetSettings();
|
||||
$redis->connect($settings['redis_host'], $settings['redis_port']);
|
||||
$redis_pwd = $settings['redis_password'];
|
||||
if (!empty($redis_pwd)) {
|
||||
$redis->auth($redis_pwd);
|
||||
}
|
||||
$redis->select($settings['redis_database']);
|
||||
$settings = $this->getSetSettings();
|
||||
$redis = $this->createRedisConnection($settings);
|
||||
$redis->rPush($settings['redis_namespace'] . ':command', 'status');
|
||||
sleep(1);
|
||||
$response = trim($redis->lPop($settings['redis_namespace'] . ':status'));
|
||||
return json_decode($response, true);
|
||||
$response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
|
||||
if ($response === null) {
|
||||
throw new Exception("No response from status command returned after 5 seconds.");
|
||||
}
|
||||
return json_decode(trim($response[1]), true);
|
||||
}
|
||||
|
||||
public function checkIfPythonLibInstalled()
|
||||
|
|
@ -92,23 +75,12 @@ class PubSubTool
|
|||
return false;
|
||||
}
|
||||
|
||||
private function __setupPubServer()
|
||||
{
|
||||
App::uses('File', 'Utility');
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$settings = $this->__getSetSettings();
|
||||
if ($this->checkIfRunning() === false) {
|
||||
shell_exec($my_server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py > ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
|
||||
}
|
||||
return $settings;
|
||||
}
|
||||
|
||||
public function publishEvent($event)
|
||||
{
|
||||
App::uses('JSONConverterTool', 'Tools');
|
||||
$jsonTool = new JSONConverterTool();
|
||||
$json = $jsonTool->convert($event);
|
||||
return $this->__pushToRedis(':data:misp_json', $json);
|
||||
return $this->pushToRedis(':data:misp_json', $json);
|
||||
}
|
||||
|
||||
public function event_save($event, $action)
|
||||
|
|
@ -116,7 +88,7 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$event['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function object_save($object, $action)
|
||||
|
|
@ -124,7 +96,7 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$object['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function object_reference_save($object_reference, $action)
|
||||
|
|
@ -132,18 +104,12 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$object_reference['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function publishConversation($message)
|
||||
{
|
||||
return $this->__pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
private function __pushToRedis($ns, $data)
|
||||
{
|
||||
$this->__redis->rPush($this->__settings['redis_namespace'] . $ns, $data);
|
||||
return true;
|
||||
return $this->pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function attribute_save($attribute, $action = false)
|
||||
|
|
@ -151,7 +117,7 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$attribute['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function tag_save($tag, $action = false)
|
||||
|
|
@ -159,7 +125,7 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$tag['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_tag', json_encode($tag, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_tag', json_encode($tag, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function sighting_save($sighting, $action = false)
|
||||
|
|
@ -167,7 +133,7 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$sighting['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function modified($data, $type, $action = false)
|
||||
|
|
@ -175,7 +141,7 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$data['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function publish($data, $type, $action = false)
|
||||
|
|
@ -183,48 +149,39 @@ class PubSubTool
|
|||
if (!empty($action)) {
|
||||
$data['action'] = $action;
|
||||
}
|
||||
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
|
||||
return $this->pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
public function killService($settings = false)
|
||||
public function killService()
|
||||
{
|
||||
$redis = new Redis();
|
||||
if ($this->checkIfRunning()) {
|
||||
if ($settings == false) {
|
||||
$settings = $this->__getSetSettings();
|
||||
}
|
||||
$redis->connect($settings['redis_host'], $settings['redis_port']);
|
||||
$redis_pwd = $settings['redis_password'];
|
||||
if (!empty($redis_pwd)) {
|
||||
$redis->auth($redis_pwd);
|
||||
}
|
||||
$redis->select($settings['redis_database']);
|
||||
$settings = $this->getSetSettings();
|
||||
$redis = $this->createRedisConnection($settings);
|
||||
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
|
||||
sleep(1);
|
||||
if ($this->checkIfRunning()) {
|
||||
// Still running.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// reload the server if it is running, if not, start it
|
||||
/**
|
||||
* Reload the server if it is running, if not, start it.
|
||||
*
|
||||
* @return bool|string
|
||||
* @throws Exception
|
||||
*/
|
||||
public function reloadServer()
|
||||
{
|
||||
if (!$this->checkIfRunning()) {
|
||||
$settings = $this->__setupPubServer();
|
||||
} else {
|
||||
$settings = $this->__getSetSettings();
|
||||
$redis = new Redis();
|
||||
$redis->connect($settings['redis_host'], $settings['redis_port']);
|
||||
$redis_pwd = $settings['redis_password'];
|
||||
if (!empty($redis_pwd)) {
|
||||
$redis->auth($redis_pwd);
|
||||
}
|
||||
$redis->select($settings['redis_database']);
|
||||
$settings = $this->getSetSettings();
|
||||
$this->saveSettingToFile($settings);
|
||||
|
||||
if ($this->checkIfRunning()) {
|
||||
$redis = $this->createRedisConnection($settings);
|
||||
$redis->rPush($settings['redis_namespace'] . ':command', 'reload');
|
||||
}
|
||||
if (!$this->checkIfRunning()) {
|
||||
} else {
|
||||
return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
|
||||
}
|
||||
return true;
|
||||
|
|
@ -237,10 +194,96 @@ class PubSubTool
|
|||
return 'Could not kill the previous instance of the ZeroMQ script.';
|
||||
}
|
||||
}
|
||||
$this->__setupPubServer();
|
||||
if (!is_numeric($this->checkIfRunning())) {
|
||||
$settings = $this->getSetSettings();
|
||||
$this->setupPubServer($settings);
|
||||
if ($this->checkIfRunning() === false) {
|
||||
return 'Failed starting the ZeroMQ script.';
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $settings
|
||||
* @throws Exception
|
||||
*/
|
||||
private function setupPubServer(array $settings)
|
||||
{
|
||||
if ($this->checkIfRunning() === false) {
|
||||
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
|
||||
// Old version is running, kill it and start again new one.
|
||||
$redis = $this->createRedisConnection($settings);
|
||||
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
|
||||
sleep(1);
|
||||
}
|
||||
|
||||
$this->saveSettingToFile($settings);
|
||||
$server = ClassRegistry::init('Server');
|
||||
shell_exec($server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
|
||||
}
|
||||
}
|
||||
|
||||
private function pushToRedis($ns, $data)
|
||||
{
|
||||
$this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $settings
|
||||
* @return Redis
|
||||
*/
|
||||
private function createRedisConnection(array $settings)
|
||||
{
|
||||
$redis = new Redis();
|
||||
$redis->connect($settings['redis_host'], $settings['redis_port']);
|
||||
$redisPassword = $settings['redis_password'];
|
||||
if (!empty($redisPassword)) {
|
||||
$redis->auth($redisPassword);
|
||||
}
|
||||
$redis->select($settings['redis_database']);
|
||||
return $redis;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $settings
|
||||
* @throws Exception
|
||||
*/
|
||||
private function saveSettingToFile(array $settings)
|
||||
{
|
||||
$settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
|
||||
$settingsFile = new File($settingFilePath, true, 0644);
|
||||
if (!$settingsFile->exists()) {
|
||||
throw new Exception("Could not create zmq config file '$settingFilePath'.");
|
||||
}
|
||||
// Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
|
||||
// config file is created under one user and then changed under other user, file must be readable and writable
|
||||
// also by group.
|
||||
chmod($settingsFile->pwd(), 0660);
|
||||
if (!$settingsFile->write(json_encode($settings))) {
|
||||
throw new Exception("Could not write zmq config file '$settingFilePath'.");
|
||||
}
|
||||
$settingsFile->close();
|
||||
}
|
||||
|
||||
private function getSetSettings()
|
||||
{
|
||||
$settings = array(
|
||||
'redis_host' => 'localhost',
|
||||
'redis_port' => '6379',
|
||||
'redis_password' => '',
|
||||
'redis_database' => '1',
|
||||
'redis_namespace' => 'mispq',
|
||||
'port' => '50000',
|
||||
'username' => null,
|
||||
'password' => null,
|
||||
);
|
||||
|
||||
foreach ($settings as $key => $setting) {
|
||||
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
|
||||
if ($temp) {
|
||||
$settings[$key] = $temp;
|
||||
}
|
||||
}
|
||||
return $settings;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ class AppModel extends Model
|
|||
{
|
||||
public $name;
|
||||
|
||||
public $loadedPubSubTool = false;
|
||||
/**
|
||||
* @var PubSubTool
|
||||
*/
|
||||
private $loadedPubSubTool;
|
||||
|
||||
public $loadedKafkaPubTool = false;
|
||||
|
||||
|
|
@ -78,7 +81,7 @@ class AppModel extends Model
|
|||
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
|
||||
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
|
||||
45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
|
||||
51 => false, 52 => false, 53 => false
|
||||
51 => false, 52 => false, 53 => false, 54 => false
|
||||
);
|
||||
|
||||
public $advanced_updates_description = array(
|
||||
|
|
@ -1389,6 +1392,9 @@ class AppModel extends Model
|
|||
$this->__addIndex('user_settings', 'setting');
|
||||
}
|
||||
break;
|
||||
case 54:
|
||||
$sqlArray[] = "ALTER TABLE `sightingdbs` MODIFY `timestamp` int(11) NOT NULL DEFAULT 0;";
|
||||
break;
|
||||
case 'fixNonEmptySharingGroupID':
|
||||
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
|
||||
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
|
||||
|
|
@ -2364,20 +2370,14 @@ class AppModel extends Model
|
|||
public function getPubSubTool()
|
||||
{
|
||||
if (!$this->loadedPubSubTool) {
|
||||
$this->loadPubSubTool();
|
||||
App::uses('PubSubTool', 'Tools');
|
||||
$pubSubTool = new PubSubTool();
|
||||
$pubSubTool->initTool();
|
||||
$this->loadedPubSubTool = $pubSubTool;
|
||||
}
|
||||
return $this->loadedPubSubTool;
|
||||
}
|
||||
|
||||
public function loadPubSubTool()
|
||||
{
|
||||
App::uses('PubSubTool', 'Tools');
|
||||
$pubSubTool = new PubSubTool();
|
||||
$pubSubTool->initTool();
|
||||
$this->loadedPubSubTool = $pubSubTool;
|
||||
return true;
|
||||
}
|
||||
|
||||
public function getElasticSearchTool()
|
||||
{
|
||||
if (!$this->elasticSearchClient) {
|
||||
|
|
|
|||
|
|
@ -401,20 +401,21 @@ class Attribute extends AppModel
|
|||
);
|
||||
|
||||
public $validFormats = array(
|
||||
'json' => array('json', 'JsonExport', 'json'),
|
||||
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
|
||||
'xml' => array('xml', 'XmlExport', 'xml'),
|
||||
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
|
||||
'snort' => array('txt', 'NidsSnortExport', 'rules'),
|
||||
'text' => array('txt', 'TextExport', 'txt'),
|
||||
'hashes' => array('txt', 'HashesExport', 'txt'),
|
||||
'yara' => array('txt', 'YaraExport', 'yara'),
|
||||
'yara-json' => array('json', 'YaraExport', 'json'),
|
||||
'rpz' => array('txt', 'RPZExport', 'rpz'),
|
||||
'csv' => array('csv', 'CsvExport', 'csv'),
|
||||
'cache' => array('txt', 'CacheExport', 'cache'),
|
||||
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
|
||||
'netfilter' => array('txt', 'NetfilterExport', 'sh')
|
||||
'cache' => array('txt', 'CacheExport', 'cache'),
|
||||
'csv' => array('csv', 'CsvExport', 'csv'),
|
||||
'hashes' => array('txt', 'HashesExport', 'txt'),
|
||||
'json' => array('json', 'JsonExport', 'json'),
|
||||
'netfilter' => array('txt', 'NetfilterExport', 'sh'),
|
||||
'opendata' => array('txt', 'OpendataExport', 'txt'),
|
||||
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
|
||||
'rpz' => array('txt', 'RPZExport', 'rpz'),
|
||||
'snort' => array('txt', 'NidsSnortExport', 'rules'),
|
||||
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
|
||||
'text' => array('txt', 'TextExport', 'txt'),
|
||||
'xml' => array('xml', 'XmlExport', 'xml'),
|
||||
'yara' => array('txt', 'YaraExport', 'yara'),
|
||||
'yara-json' => array('json', 'YaraExport', 'json')
|
||||
);
|
||||
|
||||
// FIXME we need a better way to list the defaultCategories knowing that new attribute types will continue to appear in the future. We should generate this dynamically or use a function using the default_category of the $typeDefinitions
|
||||
|
|
@ -659,20 +660,10 @@ class Attribute extends AppModel
|
|||
|
||||
private function __alterAttributeCount($event_id, $increment = true)
|
||||
{
|
||||
$event = $this->Event->find('first', array(
|
||||
'recursive' => -1,
|
||||
'conditions' => array('Event.id' => $event_id)
|
||||
));
|
||||
if (!empty($event)) {
|
||||
if ($increment) {
|
||||
$event['Event']['attribute_count'] = $event['Event']['attribute_count'] + 1;
|
||||
} else {
|
||||
$event['Event']['attribute_count'] = $event['Event']['attribute_count'] - 1;
|
||||
}
|
||||
if ($event['Event']['attribute_count'] >= 0) {
|
||||
$this->Event->save($event, array('callbacks' => false));
|
||||
}
|
||||
}
|
||||
return $this->Event->updateAll(
|
||||
array('Event.attribute_count' => $increment ? 'Event.attribute_count+1' : 'GREATEST(Event.attribute_count-1, 0)'),
|
||||
array('Event.id' => $event_id)
|
||||
);
|
||||
}
|
||||
|
||||
public function afterSave($created, $options = array())
|
||||
|
|
@ -692,7 +683,7 @@ class Attribute extends AppModel
|
|||
if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) {
|
||||
$this->__beforeSaveCorrelation($this->data['Attribute']);
|
||||
if (isset($this->data['Attribute']['event_id'])) {
|
||||
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false, $passedEvent);
|
||||
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false);
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
|
|
@ -760,7 +751,7 @@ class Attribute extends AppModel
|
|||
}
|
||||
}
|
||||
}
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst')) && strpos($this->data['Attribute']['value'], '/')) {
|
||||
$this->setCIDRList();
|
||||
}
|
||||
if ($created && isset($this->data['Attribute']['event_id']) && empty($this->data['Attribute']['skip_auto_increment'])) {
|
||||
|
|
@ -813,7 +804,7 @@ class Attribute extends AppModel
|
|||
|
||||
public function afterDelete()
|
||||
{
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst')) && strpos($this->data['Attribute']['value'], '/')) {
|
||||
$this->setCIDRList();
|
||||
}
|
||||
if (isset($this->data['Attribute']['event_id'])) {
|
||||
|
|
@ -1500,8 +1491,8 @@ class Attribute extends AppModel
|
|||
if (filter_var($parts[1], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
|
||||
// convert IPv6 address to compressed format
|
||||
$parts[1] = inet_ntop(inet_pton($value));
|
||||
$value = implode('|', $parts);
|
||||
}
|
||||
$value = implode('|', $parts);
|
||||
break;
|
||||
case 'filename|md5':
|
||||
case 'filename|sha1':
|
||||
|
|
@ -1935,67 +1926,46 @@ class Attribute extends AppModel
|
|||
return ($ip & $mask) == $subnet;
|
||||
}
|
||||
|
||||
// using Snifff's solution from http://stackoverflow.com/questions/7951061/matching-ipv6-address-to-a-cidr-subnet
|
||||
// Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
|
||||
private function __ipv6InCidr($ip, $cidr)
|
||||
{
|
||||
$ip = $this->__expandIPv6Notation($ip);
|
||||
$binaryip = $this->__inet_to_bits($ip);
|
||||
list($net, $maskbits) = explode('/', $cidr);
|
||||
$net = $this->__expandIPv6Notation($net);
|
||||
if (substr($net, -1) == ':') {
|
||||
$net .= '0';
|
||||
}
|
||||
$binarynet = $this->__inet_to_bits($net);
|
||||
$ip_net_bits = substr($binaryip, 0, $maskbits);
|
||||
$net_bits = substr($binarynet, 0, $maskbits);
|
||||
return ($ip_net_bits === $net_bits);
|
||||
}
|
||||
list($address, $netmask) = explode('/', $cidr);
|
||||
|
||||
private function __expandIPv6Notation($ip)
|
||||
{
|
||||
if (strpos($ip, '::') !== false) {
|
||||
$ip = str_replace('::', str_repeat(':0', 8 - substr_count($ip, ':')).':', $ip);
|
||||
}
|
||||
if (strpos($ip, ':') === 0) {
|
||||
$ip = '0'.$ip;
|
||||
}
|
||||
return $ip;
|
||||
}
|
||||
$bytesAddr = unpack('n*', inet_pton($address));
|
||||
$bytesTest = unpack('n*', inet_pton($ip));
|
||||
|
||||
private function __inet_to_bits($inet)
|
||||
{
|
||||
$unpacked = unpack('A16', $inet);
|
||||
$unpacked = str_split($unpacked[1]);
|
||||
$binaryip = '';
|
||||
foreach ($unpacked as $char) {
|
||||
$binaryip .= str_pad(decbin(ord($char)), 8, '0', STR_PAD_LEFT);
|
||||
for ($i = 1, $ceil = ceil($netmask / 16); $i <= $ceil; ++$i) {
|
||||
$left = $netmask - 16 * ($i - 1);
|
||||
$left = ($left <= 16) ? $left : 16;
|
||||
$mask = ~(0xffff >> $left) & 0xffff;
|
||||
if (($bytesAddr[$i] & $mask) != ($bytesTest[$i] & $mask)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return $binaryip;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function __cidrCorrelation($a)
|
||||
{
|
||||
$ipValues = array();
|
||||
$ip = $a['type'] == 'domain-ip' ? $a['value2'] : $a['value1'];
|
||||
if (strpos($ip, '/') !== false) {
|
||||
$ip = $a['value1'];
|
||||
if (strpos($ip, '/') !== false) { // IP is CIDR
|
||||
$ip_array = explode('/', $ip);
|
||||
$ip_version = filter_var($ip_array[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
$ipList = $this->find('list', array(
|
||||
'conditions' => array(
|
||||
'type' => array('ip-src', 'ip-dst', 'domain_ip'),
|
||||
'type' => array('ip-src', 'ip-dst'),
|
||||
'value1 NOT LIKE' => '%/%', // do not return CIDR, just plain IPs
|
||||
),
|
||||
'fields' => array('value1', 'value2'),
|
||||
'group' => 'value1', // return just unique values
|
||||
'fields' => array('value1'),
|
||||
'order' => false
|
||||
));
|
||||
$ipList = array_merge(array_keys($ipList), array_values($ipList));
|
||||
foreach ($ipList as $key => $value) {
|
||||
if ($value == '') {
|
||||
unset($ipList[$key]);
|
||||
}
|
||||
}
|
||||
foreach ($ipList as $ipToCheck) {
|
||||
if (filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) && $ip_version == 4) {
|
||||
if ($ip_version == 4) {
|
||||
$ipToCheckVersion = filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
if ($ipToCheckVersion === $ip_version) {
|
||||
if ($ip_version === 4) {
|
||||
if ($this->__ipv4InCidr($ipToCheck, $ip)) {
|
||||
$ipValues[] = $ipToCheck;
|
||||
}
|
||||
|
|
@ -2007,19 +1977,18 @@ class Attribute extends AppModel
|
|||
}
|
||||
}
|
||||
} else {
|
||||
$ip = $a['value1'];
|
||||
$ip_version = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
$cidrList = $this->getSetCIDRList();
|
||||
foreach ($cidrList as $cidr) {
|
||||
$cidr_ip = explode('/', $cidr)[0];
|
||||
if (filter_var($cidr_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
|
||||
if ($ip_version == 4) {
|
||||
if ($ip_version === 4) {
|
||||
if ($this->__ipv4InCidr($ip, $cidr)) {
|
||||
$ipValues[] = $cidr;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ($ip_version == 6) {
|
||||
if ($ip_version === 6) {
|
||||
if ($this->__ipv6InCidr($ip, $cidr)) {
|
||||
$ipValues[] = $cidr;
|
||||
}
|
||||
|
|
@ -2055,7 +2024,7 @@ class Attribute extends AppModel
|
|||
if (!empty($event['Event']['disable_correlation']) && $event['Event']['disable_correlation']) {
|
||||
return true;
|
||||
}
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($a['type'], array('ip-src', 'ip-dst', 'domain-ip'))) {
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($a['type'], array('ip-src', 'ip-dst'))) {
|
||||
$extraConditions = $this->__cidrCorrelation($a);
|
||||
}
|
||||
if ($a['type'] == 'ssdeep') {
|
||||
|
|
@ -3845,6 +3814,7 @@ class Attribute extends AppModel
|
|||
'value1 LIKE' => '%/%'
|
||||
),
|
||||
'fields' => array('value1'),
|
||||
'group' => 'value1', // return just unique value
|
||||
'order' => false
|
||||
));
|
||||
}
|
||||
|
|
@ -3856,12 +3826,15 @@ class Attribute extends AppModel
|
|||
if ($redis) {
|
||||
$redis->del('misp:cidr_cache_list');
|
||||
$cidrList = $this->__getCIDRList();
|
||||
$pipeline = $redis->multi(Redis::PIPELINE);
|
||||
foreach ($cidrList as $cidr) {
|
||||
$pipeline->sadd('misp:cidr_cache_list', $cidr);
|
||||
if (method_exists($redis, 'saddArray')) {
|
||||
$redis->sAddArray('misp:cidr_cache_list', $cidrList);
|
||||
} else {
|
||||
$pipeline = $redis->multi(Redis::PIPELINE);
|
||||
foreach ($cidrList as $cidr) {
|
||||
$pipeline->sadd('misp:cidr_cache_list', $cidr);
|
||||
}
|
||||
$pipeline->exec();
|
||||
}
|
||||
$pipeline->exec();
|
||||
$redis->smembers('misp:cidr_cache_list');
|
||||
}
|
||||
return $cidrList;
|
||||
}
|
||||
|
|
@ -3870,8 +3843,8 @@ class Attribute extends AppModel
|
|||
{
|
||||
$redis = $this->setupRedis();
|
||||
if ($redis) {
|
||||
if (!$redis->exists('misp:cidr_cache_list') || $redis->sCard('misp:cidr_cache_list') == 0) {
|
||||
$cidrList = $this->setCIDRList($redis);
|
||||
if ($redis->sCard('misp:cidr_cache_list') === 0) {
|
||||
$cidrList = $this->setCIDRList();
|
||||
} else {
|
||||
$cidrList = $redis->smembers('misp:cidr_cache_list');
|
||||
}
|
||||
|
|
@ -3911,7 +3884,7 @@ class Attribute extends AppModel
|
|||
'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => 'Other', 'to_ids' => 0, 'disable_correlation' => 1, 'object_relation' => 'size-in-bytes')
|
||||
);
|
||||
$hashes = array('md5', 'sha1', 'sha256');
|
||||
$this->Object = ClassRegistry::init('Object');
|
||||
$this->Object = ClassRegistry::init('MispObject');
|
||||
$this->ObjectTemplate = ClassRegistry::init('ObjectTemplate');
|
||||
$current = $this->ObjectTemplate->find('first', array(
|
||||
'fields' => array('MAX(version) AS version', 'uuid'),
|
||||
|
|
@ -4123,7 +4096,7 @@ class Attribute extends AppModel
|
|||
return $attribute;
|
||||
}
|
||||
|
||||
public function editAttribute($attribute, $eventId, $user, $objectId, $log = false)
|
||||
public function editAttribute($attribute, $eventId, $user, $objectId, $log = false, $force = false)
|
||||
{
|
||||
$attribute['event_id'] = $eventId;
|
||||
$attribute['object_id'] = $objectId;
|
||||
|
|
@ -4167,7 +4140,7 @@ class Attribute extends AppModel
|
|||
// If yes, it means that it's newer, so insert it. If no, it means that it's the same attribute or older - don't insert it, insert the old attribute.
|
||||
// Alternatively, we could unset this attribute from the request, but that could lead with issues if we decide that we want to start deleting attributes that don't exist in a pushed event.
|
||||
if (isset($attribute['timestamp'])) {
|
||||
if ($attribute['timestamp'] <= $existingAttribute['Attribute']['timestamp']) {
|
||||
if (!$force && $attribute['timestamp'] <= $existingAttribute['Attribute']['timestamp']) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -4448,6 +4421,9 @@ class Attribute extends AppModel
|
|||
}
|
||||
App::uses($this->validFormats[$returnFormat][1], 'Export');
|
||||
$exportTool = new $this->validFormats[$returnFormat][1]();
|
||||
if (!empty($exportTool->use_default_filters)) {
|
||||
$exportTool->setDefaultFilters($filters);
|
||||
}
|
||||
if (empty($exportTool->non_restrictive_export)) {
|
||||
if (!isset($filters['to_ids'])) {
|
||||
$filters['to_ids'] = 1;
|
||||
|
|
@ -4558,7 +4534,9 @@ class Attribute extends AppModel
|
|||
$loop = true;
|
||||
$params['page'] = 1;
|
||||
}
|
||||
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
|
||||
if (empty($exportTool->mock_query_only)) {
|
||||
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
|
||||
}
|
||||
fwrite($tmpfile, $exportTool->footer($exportToolParams));
|
||||
fseek($tmpfile, 0);
|
||||
if (fstat($tmpfile)['size']) {
|
||||
|
|
|
|||
|
|
@ -173,24 +173,25 @@ class Event extends AppModel
|
|||
);
|
||||
|
||||
public $validFormats = array(
|
||||
'json' => array('json', 'JsonExport', 'json'),
|
||||
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
|
||||
'xml' => array('xml', 'XmlExport', 'xml'),
|
||||
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
|
||||
'snort' => array('txt', 'NidsSnortExport', 'rules'),
|
||||
'rpz' => array('txt', 'RPZExport', 'rpz'),
|
||||
'text' => array('text', 'TextExport', 'txt'),
|
||||
'hashes' => array('txt', 'HashesExport', 'txt'),
|
||||
'attack' => array('html', 'AttackExport', 'html'),
|
||||
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
|
||||
'cache' => array('txt', 'CacheExport', 'cache'),
|
||||
'csv' => array('csv', 'CsvExport', 'csv'),
|
||||
'hashes' => array('txt', 'HashesExport', 'txt'),
|
||||
'json' => array('json', 'JsonExport', 'json'),
|
||||
'netfilter' => array('txt', 'NetfilterExport', 'sh'),
|
||||
'opendata' => array('txt', 'OpendataExport', 'txt'),
|
||||
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
|
||||
'rpz' => array('txt', 'RPZExport', 'rpz'),
|
||||
'snort' => array('txt', 'NidsSnortExport', 'rules'),
|
||||
'stix' => array('xml', 'Stix1Export', 'xml'),
|
||||
'stix-json' => array('json', 'Stix1Export', 'json'),
|
||||
'stix2' => array('json', 'Stix2Export', 'json'),
|
||||
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
|
||||
'text' => array('text', 'TextExport', 'txt'),
|
||||
'xml' => array('xml', 'XmlExport', 'xml'),
|
||||
'yara' => array('txt', 'YaraExport', 'yara'),
|
||||
'yara-json' => array('json', 'YaraExport', 'json'),
|
||||
'cache' => array('txt', 'CacheExport', 'cache'),
|
||||
'attack' => array('html', 'AttackExport', 'html'),
|
||||
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
|
||||
'netfilter' => array('txt', 'NetfilterExport', 'sh')
|
||||
'yara-json' => array('json', 'YaraExport', 'json')
|
||||
);
|
||||
|
||||
public $csv_event_context_fields_to_fetch = array(
|
||||
|
|
@ -2231,6 +2232,8 @@ class Event extends AppModel
|
|||
}
|
||||
$event = $this->__filterBlockedAttributesByTags($event, $options, $user);
|
||||
$event['Attribute'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['Attribute'], $sharingGroupData);
|
||||
|
||||
$proposalBlockAttributes = Configure::read('MISP.proposals_block_attributes');
|
||||
// move all object attributes to a temporary container
|
||||
$tempObjectAttributeContainer = array();
|
||||
foreach ($event['Attribute'] as $key => $attribute) {
|
||||
|
|
@ -2285,10 +2288,7 @@ class Event extends AppModel
|
|||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
Configure::read('MISP.proposals_block_attributes') &&
|
||||
!empty($options['allow_proposal_blocking'])
|
||||
) {
|
||||
if ($proposalBlockAttributes && !empty($options['allow_proposal_blocking'])) {
|
||||
foreach ($results[$eventKey]['Attribute'][$key]['ShadowAttribute'] as $sa) {
|
||||
if ($sa['proposal_to_delete'] || $sa['to_ids'] == 0) {
|
||||
unset($results[$eventKey]['Attribute'][$key]);
|
||||
|
|
@ -2322,15 +2322,15 @@ class Event extends AppModel
|
|||
}
|
||||
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit);
|
||||
}
|
||||
}
|
||||
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
|
||||
$this->Feed = ClassRegistry::init('Feed');
|
||||
if (!empty($options['overrideLimit'])) {
|
||||
$overrideLimit = true;
|
||||
} else {
|
||||
$overrideLimit = false;
|
||||
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
|
||||
$this->Feed = ClassRegistry::init('Feed');
|
||||
if (!empty($options['overrideLimit'])) {
|
||||
$overrideLimit = true;
|
||||
} else {
|
||||
$overrideLimit = false;
|
||||
}
|
||||
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
|
||||
}
|
||||
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
|
||||
}
|
||||
if (empty($options['metadata'])) {
|
||||
$this->Sighting = ClassRegistry::init('Sighting');
|
||||
|
|
@ -3731,7 +3731,7 @@ class Event extends AppModel
|
|||
}
|
||||
}
|
||||
|
||||
public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null)
|
||||
public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null, $force = false)
|
||||
{
|
||||
$data = $this->cleanupEventArrayFromXML($data);
|
||||
unset($this->Attribute->validate['event_id']);
|
||||
|
|
@ -3769,7 +3769,7 @@ class Event extends AppModel
|
|||
// Conditions affecting all:
|
||||
// user.org == event.org
|
||||
// edit timestamp newer than existing event timestamp
|
||||
if (!isset($data['Event']['timestamp']) || $data['Event']['timestamp'] > $existingEvent['Event']['timestamp']) {
|
||||
if ($force || !isset($data['Event']['timestamp']) || $data['Event']['timestamp'] > $existingEvent['Event']['timestamp']) {
|
||||
if (!isset($data['Event']['timestamp'])) {
|
||||
$data['Event']['timestamp'] = $date;
|
||||
}
|
||||
|
|
@ -3841,7 +3841,7 @@ class Event extends AppModel
|
|||
if (isset($data['Event']['Attribute'])) {
|
||||
$data['Event']['Attribute'] = array_values($data['Event']['Attribute']);
|
||||
foreach ($data['Event']['Attribute'] as $k => $attribute) {
|
||||
$result = $this->Attribute->editAttribute($attribute, $this->id, $user, 0, $this->Log);
|
||||
$result = $this->Attribute->editAttribute($attribute, $this->id, $user, 0, $this->Log, $force);
|
||||
if ($result !== true) {
|
||||
$validationErrors['Attribute'][] = $result;
|
||||
}
|
||||
|
|
@ -3850,7 +3850,7 @@ class Event extends AppModel
|
|||
if (isset($data['Event']['Object'])) {
|
||||
$data['Event']['Object'] = array_values($data['Event']['Object']);
|
||||
foreach ($data['Event']['Object'] as $k => $object) {
|
||||
$result = $this->Object->editObject($object, $this->id, $user, $this->Log);
|
||||
$result = $this->Object->editObject($object, $this->id, $user, $this->Log, $force);
|
||||
if ($result !== true) {
|
||||
$validationErrors['Object'][] = $result;
|
||||
}
|
||||
|
|
@ -3858,7 +3858,7 @@ class Event extends AppModel
|
|||
foreach ($data['Event']['Object'] as $object) {
|
||||
if (isset($object['ObjectReference'])) {
|
||||
foreach ($object['ObjectReference'] as $objectRef) {
|
||||
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log);
|
||||
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log, $force);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -6680,6 +6680,10 @@ class Event extends AppModel
|
|||
$this->Job->id = $jobId;
|
||||
}
|
||||
|
||||
if (!empty($exportTool->use_default_filters)) {
|
||||
$exportTool->setDefaultFilters($filters);
|
||||
}
|
||||
|
||||
if (empty($exportTool->non_restrictive_export)) {
|
||||
if (!isset($filters['to_ids'])) {
|
||||
$filters['to_ids'] = 1;
|
||||
|
|
@ -6719,11 +6723,15 @@ class Event extends AppModel
|
|||
$subqueryElements = $this->harvestSubqueryElements($filters);
|
||||
$filters = $this->addFiltersFromSubqueryElements($filters, $subqueryElements);
|
||||
|
||||
$filters['include_attribute_count'] = 1;
|
||||
$eventid = $this->filterEventIds($user, $filters, $elementCounter);
|
||||
$eventCount = count($eventid);
|
||||
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
|
||||
unset($eventid);
|
||||
if (empty($exportTool->mock_query_only)) {
|
||||
$filters['include_attribute_count'] = 1;
|
||||
$eventid = $this->filterEventIds($user, $filters, $elementCounter);
|
||||
$eventCount = count($eventid);
|
||||
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
|
||||
unset($eventid);
|
||||
} else {
|
||||
$eventids_chunked = array();
|
||||
}
|
||||
if (!empty($exportTool->additional_params)) {
|
||||
$filters = array_merge($filters, $exportTool->additional_params);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ class Feed extends AppModel
|
|||
private function downloadManifest($feed, $HttpSocket)
|
||||
{
|
||||
$manifestUrl = $feed['Feed']['url'] . '/manifest.json';
|
||||
$data = $this->feedGetUri($feed, $manifestUrl, $HttpSocket);
|
||||
$data = $this->feedGetUri($feed, $manifestUrl, $HttpSocket, true);
|
||||
|
||||
$manifest = json_decode($data, true);
|
||||
if ($manifest === null) {
|
||||
|
|
@ -328,109 +328,128 @@ class Feed extends AppModel
|
|||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach correlations from cached servers or feeds.
|
||||
*
|
||||
* @param array $objects
|
||||
* @param array $user
|
||||
* @param array $event
|
||||
* @param bool $overrideLimit Override hardcoded limit for 10 000 attribute correlations.
|
||||
* @param string $scope `Feed` or `Server`
|
||||
* @return array
|
||||
*/
|
||||
public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false, $scope = 'Feed')
|
||||
{
|
||||
$redis = $this->setupRedis();
|
||||
if ($redis !== false) {
|
||||
$pipe = $redis->multi(Redis::PIPELINE);
|
||||
$hashTable = array();
|
||||
$cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
|
||||
try {
|
||||
$redis = $this->setupRedisWithException();
|
||||
} catch (Exception $e) {
|
||||
return $objects;
|
||||
}
|
||||
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$compositeTypes = $this->Event->Attribute->getCompositeTypes();
|
||||
$cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
|
||||
|
||||
foreach ($objects as $k => $object) {
|
||||
if (in_array($object['type'], $compositeTypes)) {
|
||||
$value = explode('|', $object['value']);
|
||||
$hashTable[$k] = md5($value[0]);
|
||||
} else {
|
||||
$hashTable[$k] = md5($object['value']);
|
||||
}
|
||||
$redis->sismember($cachePrefix . 'combined', $hashTable[$k]);
|
||||
}
|
||||
$results = $pipe->exec();
|
||||
if (!$overrideLimit && count($objects) > 10000) {
|
||||
foreach ($results as $k => $result) {
|
||||
if ($result && empty($objects[$k]['disable_correlation'])) {
|
||||
if (isset($event['FeedCount'])) {
|
||||
$event['FeedCount']++;
|
||||
} else {
|
||||
$event['FeedCount'] = 1;
|
||||
}
|
||||
$objects[$k]['FeedHit'] = true;
|
||||
}
|
||||
}
|
||||
// Redis cache for $scope is empty.
|
||||
if ($redis->sCard($cachePrefix . 'combined') === 0) {
|
||||
return $objects;
|
||||
}
|
||||
|
||||
$pipe = $redis->multi(Redis::PIPELINE);
|
||||
$hashTable = array();
|
||||
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$compositeTypes = $this->Event->Attribute->getCompositeTypes();
|
||||
|
||||
foreach ($objects as $k => $object) {
|
||||
if (in_array($object['type'], $compositeTypes)) {
|
||||
$value = explode('|', $object['value']);
|
||||
$hashTable[$k] = md5($value[0]);
|
||||
} else {
|
||||
if ($scope === 'Feed') {
|
||||
$params = array(
|
||||
'recursive' => -1,
|
||||
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
|
||||
);
|
||||
if (!$user['Role']['perm_site_admin']) {
|
||||
$params['conditions'] = array('Feed.lookup_visible' => 1);
|
||||
$hashTable[$k] = md5($object['value']);
|
||||
}
|
||||
$redis->sismember($cachePrefix . 'combined', $hashTable[$k]);
|
||||
}
|
||||
$results = $pipe->exec();
|
||||
if (!$overrideLimit && count($objects) > 10000) {
|
||||
foreach ($results as $k => $result) {
|
||||
if ($result && empty($objects[$k]['disable_correlation'])) {
|
||||
if (isset($event['FeedCount'])) {
|
||||
$event['FeedCount']++;
|
||||
} else {
|
||||
$event['FeedCount'] = 1;
|
||||
}
|
||||
$sources = $this->find('all', $params);
|
||||
} else {
|
||||
$params = array(
|
||||
'recursive' => -1,
|
||||
'fields' => array('id', 'name', 'url', 'caching_enabled')
|
||||
);
|
||||
if (!$user['Role']['perm_site_admin']) {
|
||||
$params['conditions'] = array('Server.caching_enabled' => 1);
|
||||
}
|
||||
$this->Server = ClassRegistry::init('Server');
|
||||
$sources = $this->Server->find('all', $params);
|
||||
$objects[$k]['FeedHit'] = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ($scope === 'Feed') {
|
||||
$params = array(
|
||||
'recursive' => -1,
|
||||
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
|
||||
);
|
||||
if (!$user['Role']['perm_site_admin']) {
|
||||
$params['conditions'] = array('Feed.lookup_visible' => 1);
|
||||
}
|
||||
$sources = $this->find('all', $params);
|
||||
} else {
|
||||
$params = array(
|
||||
'recursive' => -1,
|
||||
'fields' => array('id', 'name', 'url', 'caching_enabled')
|
||||
);
|
||||
if (!$user['Role']['perm_site_admin']) {
|
||||
$params['conditions'] = array('Server.caching_enabled' => 1);
|
||||
}
|
||||
$this->Server = ClassRegistry::init('Server');
|
||||
$sources = $this->Server->find('all', $params);
|
||||
}
|
||||
|
||||
$hitIds = array();
|
||||
foreach ($results as $k => $result) {
|
||||
if ($result && empty($objects[$k]['disable_correlation'])) {
|
||||
$hitIds[] = $k;
|
||||
$hitIds = array();
|
||||
foreach ($results as $k => $result) {
|
||||
if ($result && empty($objects[$k]['disable_correlation'])) {
|
||||
$hitIds[] = $k;
|
||||
}
|
||||
}
|
||||
foreach ($sources as $source) {
|
||||
$sourceScopeId = $source[$scope]['id'];
|
||||
|
||||
$pipe = $redis->multi(Redis::PIPELINE);
|
||||
foreach ($hitIds as $k) {
|
||||
$redis->sismember($cachePrefix . $sourceScopeId, $hashTable[$k]);
|
||||
}
|
||||
$sourceHits = $pipe->exec();
|
||||
foreach ($sourceHits as $k4 => $hit) {
|
||||
if ($hit) {
|
||||
if (!isset($event[$scope][$sourceScopeId]['id'])) {
|
||||
if (!isset($event[$scope][$sourceScopeId])) {
|
||||
$event[$scope][$sourceScopeId] = array();
|
||||
}
|
||||
$event[$scope][$sourceScopeId] = array_merge($event[$scope][$sourceScopeId], $source[$scope]);
|
||||
}
|
||||
$objects[$hitIds[$k4]][$scope][] = $source[$scope];
|
||||
}
|
||||
}
|
||||
foreach ($sources as $source) {
|
||||
$sourceScopeId = $source[$scope]['id'];
|
||||
|
||||
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
|
||||
$pipe = $redis->multi(Redis::PIPELINE);
|
||||
foreach ($hitIds as $k) {
|
||||
$redis->sismember($cachePrefix . $sourceScopeId, $hashTable[$k]);
|
||||
}
|
||||
$sourceHits = $pipe->exec();
|
||||
foreach ($sourceHits as $k4 => $hit) {
|
||||
if ($hit) {
|
||||
if (!isset($event[$scope][$sourceScopeId]['id'])) {
|
||||
if (!isset($event[$scope][$sourceScopeId])) {
|
||||
$event[$scope][$sourceScopeId] = array();
|
||||
}
|
||||
$event[$scope][$sourceScopeId] = array_merge($event[$scope][$sourceScopeId], $source[$scope]);
|
||||
}
|
||||
$objects[$hitIds[$k4]][$scope][] = $source[$scope];
|
||||
}
|
||||
}
|
||||
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
|
||||
$pipe = $redis->multi(Redis::PIPELINE);
|
||||
$eventUuidHitPosition = array();
|
||||
foreach ($objects as $k => $object) {
|
||||
if (isset($object[$scope])) {
|
||||
foreach ($object[$scope] as $currentFeed) {
|
||||
if ($source[$scope]['id'] == $currentFeed['id']) {
|
||||
$eventUuidHitPosition[] = $k;
|
||||
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
|
||||
}
|
||||
$eventUuidHitPosition = array();
|
||||
foreach ($objects as $k => $object) {
|
||||
if (isset($object[$scope])) {
|
||||
foreach ($object[$scope] as $currentFeed) {
|
||||
if ($source[$scope]['id'] == $currentFeed['id']) {
|
||||
$eventUuidHitPosition[] = $k;
|
||||
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
|
||||
}
|
||||
}
|
||||
}
|
||||
$mispFeedHits = $pipe->exec();
|
||||
foreach ($mispFeedHits as $sourcehitPos => $f) {
|
||||
foreach ($f as $url) {
|
||||
list($feedId, $eventUuid) = explode('/', $url);
|
||||
if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
|
||||
$event[$scope][$feedId]['event_uuids'][] = $eventUuid;
|
||||
}
|
||||
foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
|
||||
if ($tempFeed['id'] == $feedId) {
|
||||
$objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $eventUuid;
|
||||
}
|
||||
}
|
||||
$mispFeedHits = $pipe->exec();
|
||||
foreach ($mispFeedHits as $sourcehitPos => $f) {
|
||||
foreach ($f as $url) {
|
||||
list($feedId, $eventUuid) = explode('/', $url);
|
||||
if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
|
||||
$event[$scope][$feedId]['event_uuids'][] = $eventUuid;
|
||||
}
|
||||
foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
|
||||
if ($tempFeed['id'] == $feedId) {
|
||||
$objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $eventUuid;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -438,9 +457,11 @@ class Feed extends AppModel
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($event[$scope])) {
|
||||
$event[$scope] = array_values($event[$scope]);
|
||||
}
|
||||
|
||||
return $objects;
|
||||
}
|
||||
|
||||
|
|
@ -846,6 +867,10 @@ class Feed extends AppModel
|
|||
return false;
|
||||
}
|
||||
|
||||
if (empty($temp)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$data = array();
|
||||
foreach ($temp as $value) {
|
||||
$data[] = array(
|
||||
|
|
@ -855,14 +880,11 @@ class Feed extends AppModel
|
|||
'to_ids' => $value['to_ids']
|
||||
);
|
||||
}
|
||||
if (empty($data)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$this->jobProgress($jobId, 'Saving data.', 50);
|
||||
|
||||
try {
|
||||
$result = $this->saveFreetextFeedData($this->data, $data, $user);
|
||||
$result = $this->saveFreetextFeedData($this->data, $data, $user, $jobId);
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Could not save freetext feed data for feed $feedId.", $e);
|
||||
return false;
|
||||
|
|
@ -936,7 +958,7 @@ class Feed extends AppModel
|
|||
}
|
||||
}
|
||||
if ($feed['Feed']['fixed_event']) {
|
||||
$temp = $this->Event->Attribute->find('all', array(
|
||||
$existsAttributes = $this->Event->Attribute->find('all', array(
|
||||
'conditions' => array(
|
||||
'Attribute.deleted' => 0,
|
||||
'Attribute.event_id' => $event['Event']['id']
|
||||
|
|
@ -944,28 +966,38 @@ class Feed extends AppModel
|
|||
'recursive' => -1,
|
||||
'fields' => array('id', 'value1', 'value2')
|
||||
));
|
||||
$event['Attribute'] = array();
|
||||
foreach ($temp as $t) {
|
||||
$existsAttributesValueToId = array();
|
||||
foreach ($existsAttributes as $t) {
|
||||
if (!empty($t['Attribute']['value2'])) {
|
||||
$value = $t['Attribute']['value1'] . '|' . $t['Attribute']['value2'];
|
||||
} else {
|
||||
$value = $t['Attribute']['value1'];
|
||||
}
|
||||
$event['Attribute'][$t['Attribute']['id']] = $value;
|
||||
// Since event values are unique, it is OK to put value into key
|
||||
$existsAttributesValueToId[$value] = $t['Attribute']['id'];
|
||||
}
|
||||
unset($temp);
|
||||
unset($existsAttributes);
|
||||
|
||||
// Create event diff. After this cycle, `$data` will contains just attributes that do not exists in current
|
||||
// event and in `$existsAttributesValueToId` will contains just attributes that do not exists in current feed.
|
||||
foreach ($data as $k => $dataPoint) {
|
||||
$finder = array_search($dataPoint['value'], $event['Attribute']);
|
||||
if ($finder !== false) {
|
||||
if (isset($existsAttributesValueToId[$dataPoint['value']])) {
|
||||
unset($data[$k]);
|
||||
unset($event['Attribute'][$finder]);
|
||||
unset($existsAttributesValueToId[$dataPoint['value']]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Because some types can be saved in modified version (for example, IPv6 address is convert to compressed
|
||||
// format, we should also check if current event contains modified value.
|
||||
$modifiedValue = $this->Event->Attribute->modifyBeforeValidation($dataPoint['type'], $dataPoint['value']);
|
||||
if (isset($existsAttributesValueToId[$modifiedValue])) {
|
||||
unset($data[$k]);
|
||||
unset($existsAttributesValueToId[$modifiedValue]);
|
||||
}
|
||||
}
|
||||
if ($feed['Feed']['delta_merge']) {
|
||||
$to_delete = array_keys($event['Attribute']);
|
||||
if (!empty($to_delete)) {
|
||||
$this->Event->Attribute->deleteAll(array('Attribute.id' => $to_delete, 'Attribute.deleted' => 0));
|
||||
}
|
||||
if ($feed['Feed']['delta_merge'] && !empty($existsAttributesValueToId)) {
|
||||
$to_delete = array_values($existsAttributesValueToId);
|
||||
$this->Event->Attribute->deleteAll(array('Attribute.id' => $to_delete, 'Attribute.deleted' => 0));
|
||||
}
|
||||
}
|
||||
if (empty($data)) {
|
||||
|
|
@ -989,8 +1021,8 @@ class Feed extends AppModel
|
|||
foreach ($data as $k => $chunk) {
|
||||
$this->Event->Attribute->create();
|
||||
$this->Event->Attribute->save($chunk);
|
||||
if ($k % 100 == 0) {
|
||||
$this->jobProgress($jobId, null, 50 + round((50 * ((($k + 1) * 100) / count($data)))));
|
||||
if ($k % 100 === 0) {
|
||||
$this->jobProgress($jobId, null, 50 + round(($k + 1) / count($data) * 50));
|
||||
}
|
||||
}
|
||||
if (!empty($data)) {
|
||||
|
|
|
|||
|
|
@ -67,26 +67,19 @@ class FuzzyCorrelateSsdeep extends AppModel
|
|||
|
||||
public function query_ssdeep_chunks($hash, $attribute_id)
|
||||
{
|
||||
//'12288:CeqW86Tf7xglFIV/4Zf8FkKBPFrmtJxv/znLABkeGevRcAqn9LqgqmlrexDvBIRF:CV6fxg7IeEOKXrmtJx3rLABk1eFElree'
|
||||
$chunks = $this->ssdeep_prepare($hash);
|
||||
// if chunk_size == chunk_size_1 OR 2*chunk_size == chunk_size
|
||||
// SELECT * from ssdeep_chunks where (chunk_size = $chunk_size OR chunk_size*2 = $chunk_size)
|
||||
// AND chunk_size
|
||||
|
||||
// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
|
||||
// also propose to insert chunk size to database, but current database schema doesn't contain that column.
|
||||
// This optimisation can be add in future versions.
|
||||
$result = $this->find('list', array(
|
||||
'conditions' => array(
|
||||
'AND' => array(
|
||||
'OR' => array(
|
||||
'FuzzyCorrelateSsdeep.chunk_size' => $chunks[0],
|
||||
'FuzzyCorrelateSsdeep.chunk_size' => $chunks[0] * 2,
|
||||
),
|
||||
'OR' => array(
|
||||
'FuzzyCorrelateSsdeep.chunk' => $chunks[1],
|
||||
'FuzzyCorrelateSsdeep.chunk' => $chunks[2]
|
||||
)
|
||||
)
|
||||
'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]),
|
||||
),
|
||||
'fields' => array('FuzzyCorrelateSsdeep.attribute_id', 'FuzzyCorrelateSsdeep.attribute_id')
|
||||
'fields' => array('FuzzyCorrelateSsdeep.attribute_id', 'FuzzyCorrelateSsdeep.attribute_id'),
|
||||
'group' => 'FuzzyCorrelateSsdeep.attribute_id', // remove duplicates at database side
|
||||
));
|
||||
|
||||
$to_save = array();
|
||||
foreach (array(1, 2) as $type) {
|
||||
foreach ($chunks[$type] as $chunk) {
|
||||
|
|
|
|||
|
|
@ -196,7 +196,8 @@ class Log extends AppModel
|
|||
if (is_array($change)) {
|
||||
$output = array();
|
||||
foreach ($change as $field => $values) {
|
||||
if (strpos($field, 'password') !== false) { // if field name contains password, replace value with asterisk
|
||||
$isSecret = strpos($field, 'password') !== false || ($field === 'authkey' && Configure::read('Security.do_not_log_authkeys'));
|
||||
if ($isSecret) {
|
||||
$oldValue = $newValue = "*****";
|
||||
} else {
|
||||
list($oldValue, $newValue) = $values;
|
||||
|
|
|
|||
|
|
@ -957,7 +957,7 @@ class MispObject extends AppModel
|
|||
return 'fail';
|
||||
}
|
||||
|
||||
public function editObject($object, $eventId, $user, $log)
|
||||
public function editObject($object, $eventId, $user, $log, $force = false)
|
||||
{
|
||||
$object['event_id'] = $eventId;
|
||||
if (isset($object['uuid'])) {
|
||||
|
|
@ -983,7 +983,7 @@ class MispObject extends AppModel
|
|||
return true;
|
||||
}
|
||||
if (isset($object['timestamp'])) {
|
||||
if ($existingObject['Object']['timestamp'] >= $object['timestamp']) {
|
||||
if ($force || $existingObject['Object']['timestamp'] >= $object['timestamp']) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1033,7 +1033,7 @@ class MispObject extends AppModel
|
|||
}
|
||||
if (!empty($object['Attribute'])) {
|
||||
foreach ($object['Attribute'] as $attribute) {
|
||||
$result = $this->Attribute->editAttribute($attribute, $eventId, $user, $object['id'], $log);
|
||||
$result = $this->Attribute->editAttribute($attribute, $eventId, $user, $object['id'], $log, $force);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -1261,6 +1261,15 @@ class Server extends AppModel
|
|||
'type' => 'boolean',
|
||||
'null' => true
|
||||
),
|
||||
'do_not_log_authkeys' => array(
|
||||
'level' => 0,
|
||||
'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'),
|
||||
'value' => false,
|
||||
'errorMessage' => '',
|
||||
'test' => 'testBool',
|
||||
'type' => 'boolean',
|
||||
'null' => true
|
||||
),
|
||||
'email_otp_enabled' => array(
|
||||
'level'=> 2,
|
||||
'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'),
|
||||
|
|
@ -1806,6 +1815,24 @@ class Server extends AppModel
|
|||
'type' => 'numeric',
|
||||
'afterHook' => 'zmqAfterHook',
|
||||
),
|
||||
'ZeroMQ_username' => array(
|
||||
'level' => 2,
|
||||
'description' => __('The username that client need to use to connect to ZeroMQ.'),
|
||||
'value' => '',
|
||||
'errorMessage' => '',
|
||||
'test' => 'testForEmpty',
|
||||
'type' => 'string',
|
||||
'afterHook' => 'zmqAfterHook',
|
||||
),
|
||||
'ZeroMQ_password' => array(
|
||||
'level' => 2,
|
||||
'description' => __('The password that client need to use to connect to ZeroMQ.'),
|
||||
'value' => '',
|
||||
'errorMessage' => '',
|
||||
'test' => 'testForEmpty',
|
||||
'type' => 'string',
|
||||
'afterHook' => 'zmqAfterHook',
|
||||
),
|
||||
'ZeroMQ_redis_host' => array(
|
||||
'level' => 2,
|
||||
'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'),
|
||||
|
|
@ -2364,11 +2391,11 @@ class Server extends AppModel
|
|||
return true;
|
||||
}
|
||||
|
||||
private function __getEventIdListBasedOnPullTechnique($technique, $server)
|
||||
private function __getEventIdListBasedOnPullTechnique($technique, $server, $force = false)
|
||||
{
|
||||
if ("full" === $technique) {
|
||||
// get a list of the event_ids on the server
|
||||
$eventIds = $this->getEventIdsFromServer($server);
|
||||
$eventIds = $this->getEventIdsFromServer($server, false, null, false, false, 'events', $force);
|
||||
if ($eventIds === 403) {
|
||||
return array('error' => array(1, null));
|
||||
} elseif (is_string($eventIds)) {
|
||||
|
|
@ -2380,7 +2407,7 @@ class Server extends AppModel
|
|||
$eventIds = array_reverse($eventIds);
|
||||
}
|
||||
} elseif ("update" === $technique) {
|
||||
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true);
|
||||
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true, 'events', $force);
|
||||
if ($eventIds === 403) {
|
||||
return array('error' => array(1, null));
|
||||
} elseif (is_string($eventIds)) {
|
||||
|
|
@ -2475,7 +2502,7 @@ class Server extends AppModel
|
|||
return false;
|
||||
}
|
||||
|
||||
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId)
|
||||
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false)
|
||||
{
|
||||
// check if the event already exist (using the uuid)
|
||||
$existingEvent = $eventModel->find('first', array('conditions' => array('Event.uuid' => $event['Event']['uuid'])));
|
||||
|
|
@ -2492,7 +2519,7 @@ class Server extends AppModel
|
|||
if (!$existingEvent['Event']['locked'] && !$server['Server']['internal']) {
|
||||
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
|
||||
} else {
|
||||
$result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong);
|
||||
$result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force);
|
||||
if ($result === true) {
|
||||
$successes[] = $eventId;
|
||||
} elseif (isset($result['error'])) {
|
||||
|
|
@ -2504,7 +2531,7 @@ class Server extends AppModel
|
|||
}
|
||||
}
|
||||
|
||||
private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId)
|
||||
private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false)
|
||||
{
|
||||
$event = $eventModel->downloadEventFromServer(
|
||||
$eventId,
|
||||
|
|
@ -2519,7 +2546,7 @@ class Server extends AppModel
|
|||
if (!$this->__checkIfEventSaveAble($event)) {
|
||||
$fails[$eventId] = __('Empty event detected.');
|
||||
} else {
|
||||
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $server, $user, $jobId);
|
||||
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $server, $user, $jobId, $force);
|
||||
}
|
||||
} else {
|
||||
// error
|
||||
|
|
@ -2584,7 +2611,7 @@ class Server extends AppModel
|
|||
return $pulledProposals;
|
||||
}
|
||||
|
||||
public function pull($user, $id = null, $technique=false, $server, $jobId = false)
|
||||
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
|
||||
{
|
||||
if ($jobId) {
|
||||
$job = ClassRegistry::init('Job');
|
||||
|
|
@ -2598,7 +2625,7 @@ class Server extends AppModel
|
|||
$eventIds = array();
|
||||
// if we are downloading a single event, don't fetch all proposals
|
||||
$conditions = is_numeric($technique) ? array('Event.id' => $technique) : array();
|
||||
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
|
||||
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server, $force);
|
||||
$server['Server']['version'] = $this->getRemoteVersion($id);
|
||||
if (!empty($eventIds['error'])) {
|
||||
$errors = array(
|
||||
|
|
@ -2627,7 +2654,7 @@ class Server extends AppModel
|
|||
if (!empty($eventIds)) {
|
||||
// download each event
|
||||
foreach ($eventIds as $k => $eventId) {
|
||||
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $server, $user, $jobId);
|
||||
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $server, $user, $jobId, $force);
|
||||
if ($jobId) {
|
||||
if ($k % 10 == 0) {
|
||||
$job->saveField('progress', 50 * (($k + 1) / count($eventIds)));
|
||||
|
|
@ -2740,7 +2767,7 @@ class Server extends AppModel
|
|||
}
|
||||
|
||||
// Get an array of event_ids that are present on the remote server
|
||||
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events')
|
||||
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events', $force = false)
|
||||
{
|
||||
$url = $server['Server']['url'];
|
||||
if ($ignoreFilterRules) {
|
||||
|
|
@ -2824,7 +2851,9 @@ class Server extends AppModel
|
|||
}
|
||||
}
|
||||
}
|
||||
$this->Event->removeOlder($eventArray, $scope);
|
||||
if (!$force) {
|
||||
$this->Event->removeOlder($eventArray, $scope);
|
||||
}
|
||||
if (!empty($eventArray)) {
|
||||
foreach ($eventArray as $event) {
|
||||
if ($force_uuid) {
|
||||
|
|
@ -4601,26 +4630,28 @@ class Server extends AppModel
|
|||
switch($field['error_type']) {
|
||||
case 'missing_column':
|
||||
$field['sql'] = sprintf(
|
||||
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s;',
|
||||
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s %s;',
|
||||
$table,
|
||||
$field['column_name'],
|
||||
$field['expected']['data_type'],
|
||||
$length !== null ? sprintf('(%d)', $length) : '',
|
||||
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
|
||||
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
|
||||
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
|
||||
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'],
|
||||
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
|
||||
);
|
||||
break;
|
||||
case 'column_different':
|
||||
$field['sql'] = sprintf(
|
||||
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s;',
|
||||
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s %s;',
|
||||
$table,
|
||||
$field['column_name'],
|
||||
$field['expected']['data_type'],
|
||||
$length !== null ? sprintf('(%d)', $length) : '',
|
||||
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
|
||||
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
|
||||
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
|
||||
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'],
|
||||
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
|
@ -4637,13 +4668,14 @@ class Server extends AppModel
|
|||
} elseif ($expectedField['data_type'] === 'text') {
|
||||
$length = null;
|
||||
}
|
||||
$fieldSql = sprintf('`%s` %s%s %s %s %s',
|
||||
$fieldSql = sprintf('`%s` %s%s %s %s %s %s',
|
||||
$expectedField['column_name'],
|
||||
$expectedField['data_type'],
|
||||
$length !== null ? sprintf('(%d)', $length) : '',
|
||||
isset($expectedField['column_default']) ? 'DEFAULT "' . $expectedField['column_default'] . '"' : '',
|
||||
$expectedField['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
|
||||
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name']
|
||||
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name'],
|
||||
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
|
||||
);
|
||||
$allFields[] = $fieldSql;
|
||||
}
|
||||
|
|
@ -4694,7 +4726,8 @@ class Server extends AppModel
|
|||
// 'datetime_precision', -- Only available on MySQL 5.6+
|
||||
'collation_name',
|
||||
'column_type',
|
||||
'column_default'
|
||||
'column_default',
|
||||
'extra',
|
||||
)
|
||||
){
|
||||
$dbActualSchema = array();
|
||||
|
|
@ -4976,7 +5009,7 @@ class Server extends AppModel
|
|||
public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion)
|
||||
{
|
||||
$result = array();
|
||||
$expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.21', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
|
||||
$expected = array('stix' => '>1.2.0.9', 'cybox' => '>2.1.0.21', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
|
||||
// check if the STIX and Cybox libraries are working using the test script stixtest.py
|
||||
$scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
|
||||
$scriptResult = json_decode($scriptResult, true);
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class Warninglist extends AppModel
|
|||
public function update()
|
||||
{
|
||||
$directories = glob(APP . 'files' . DS . 'warninglists' . DS . 'lists' . DS . '*', GLOB_ONLYDIR);
|
||||
$updated = array();
|
||||
$updated = array('success' => [], 'fails' => []);
|
||||
foreach ($directories as $dir) {
|
||||
$file = new File($dir . DS . 'list.json');
|
||||
$list = json_decode($file->read(), true);
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ class Whitelist extends AppModel
|
|||
|
||||
public function getBlockedValues()
|
||||
{
|
||||
if ($this->whitelistedItems !== false) {
|
||||
if ($this->whitelistedItems === false) {
|
||||
$Whitelists = $this->find('all', array('fields' => array('name')));
|
||||
$this->whitelistedItems = array();
|
||||
foreach ($Whitelists as $item) {
|
||||
|
|
|
|||
|
|
@ -433,6 +433,10 @@ class LogableBehavior extends ModelBehavior {
|
|||
$old = '';
|
||||
}
|
||||
if ($key != 'modified' && !in_array($key, $this->settings[$Model->alias]['ignore']) && $value != $old && in_array($key, $db_fields)) {
|
||||
if ($key === 'authkey' && Configure::read('Security.do_not_log_authkeys')) {
|
||||
$old = $value = '*****';
|
||||
}
|
||||
|
||||
if ($this->settings[$Model->alias]['change'] == 'full') {
|
||||
$changed_fields[] = $key . ' (' . $old . ') => (' . $value . ')';
|
||||
} else if ($this->settings[$Model->alias]['change'] == 'serialize') {
|
||||
|
|
|
|||
|
|
@ -66,14 +66,18 @@ class SysLogLogableBehavior extends LogableBehavior {
|
|||
if (isset($Model->data[$Model->alias][$Model->primaryKey]) && !empty($this->old) && isset($this->old[$Model->alias][$key])) {
|
||||
$old = $this->old[$Model->alias][$key];
|
||||
if (is_array($old)) {
|
||||
$old = json_encode($old, true);
|
||||
$old = json_encode($old);
|
||||
}
|
||||
} else {
|
||||
$old = '';
|
||||
}
|
||||
// TODO Audit, removed 'revision' as well
|
||||
if ($key != 'lastpushedid' && $key!= 'timestamp' && $key != 'revision' && $key != 'modified' && !in_array($key, $this->settings[$Model->alias]['ignore']) && $value != $old && in_array($key, $db_fields)) {
|
||||
if ($this->settings[$Model->alias]['change'] == 'full') {
|
||||
if ($key === 'authkey' && Configure::read('Security.do_not_log_authkeys')) {
|
||||
$old = $value = '*****';
|
||||
}
|
||||
|
||||
if ($this->settings[$Model->alias]['change'] == 'full') {
|
||||
if (($key != 'published') || (($key == 'published') && ($value == '1'))) { // remove (un-)published from edit
|
||||
$changed_fields[] = $key . ' (' . $old . ') => (' . $value . ')';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,12 +10,12 @@
|
|||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td rowspan="2" style="border-right: 1px solid #ddd; padding-right: 2px; max-width: 24px; overflow: hidden; font-size: xx-small; text-overflow: ellipsis;" title="<?php echo h($related['Orgc']['name']); ?>">
|
||||
<td rowspan="2" style="border-right: 1px solid #ddd; padding-right: 2px; min-width: 24px; max-width: 24px; overflow: hidden; font-size: xx-small; text-overflow: ellipsis;" title="<?php echo h($related['Orgc']['name']); ?>">
|
||||
<?php echo $this->OrgImg->getOrgImg(array('name' => $related['Orgc']['name'], 'id' => $related['Orgc']['id'], 'size' => 24)); ?>
|
||||
</td>
|
||||
<td style="line-height: 14px; padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 430px;">
|
||||
<td style="line-height: 14px; padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 410px;">
|
||||
<a title="<?php echo h($related['info']); ?>" href="<?php echo h($href_url)?>">
|
||||
<span><?php echo h($related['info']) ?>
|
||||
<?php echo h($related['info']) ?>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
<td style="line-height: 14px; padding-left: 2px;">
|
||||
<i><?php echo h($related['date']); ?></i>
|
||||
<?php if (isset($relatedEventCorrelationCount[$related['id']])): ?>
|
||||
<b style="margin-left: 5px; float: right; cursor: help;" title="<?php echo sprintf(__('This related event contains %s unique correlation(s)'), h($relatedEventCorrelationCount[$related['id']])); ?>"> <?php echo h($relatedEventCorrelationCount[$related['id']]) ?></b>
|
||||
<b style="margin-left: 5px; float: right; cursor: help;" title="<?php echo __('This related event contains %s unique correlation(s)', h($relatedEventCorrelationCount[$related['id']])); ?>"> <?php echo h($relatedEventCorrelationCount[$related['id']]) ?></b>
|
||||
<?php endif; ?>
|
||||
</td>
|
||||
</tr>
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
<h3><?= __('Discussion') ?></h3>
|
||||
<div id="top">
|
||||
<div class="pagination">
|
||||
<?php
|
||||
|
|
@ -18,12 +19,11 @@
|
|||
?>
|
||||
</ul>
|
||||
</div>
|
||||
<div id = "posts">
|
||||
<div id="posts">
|
||||
<?php
|
||||
foreach ($posts as $post) {
|
||||
?>
|
||||
<a name="message_<?php echo h($post['id']);?>"></a>
|
||||
<table class="discussionBox" id=<?php echo '"' . h($post['id']) . '"';?> >
|
||||
<table class="discussionBox" id="message_<?= h($post['id']) ?>">
|
||||
<tr>
|
||||
<td class="discussionBoxTD discussionBoxTDtop" colspan="2">
|
||||
<div>
|
||||
|
|
@ -36,7 +36,7 @@
|
|||
</td>
|
||||
<td style="text-align:right">
|
||||
<a href="#top" class="whitelink">Top</a> |
|
||||
<a href="<?php echo "#".$post['id']; ?>" class="whitelink">#<?php echo h($post['id'])?></a>
|
||||
<a href="#message_<?php echo h($post['id']); ?>" class="whitelink">#<?php echo h($post['id'])?></a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
@ -58,14 +58,14 @@
|
|||
echo $this->Command->convertQuotes(nl2br(h($post['contents'])));
|
||||
if ($post['post_id'] !=0 || ($post['date_created'] != $post['date_modified'])) {
|
||||
?>
|
||||
<br /><br />
|
||||
<br><br>
|
||||
<?php
|
||||
}
|
||||
if ($post['post_id'] != 0) {
|
||||
?>
|
||||
<span style="font-style:italic">
|
||||
In reply to post
|
||||
<a href="<?php echo "#".h($post['post_id']); ?>">#<?php echo h($post['post_id'])?></a>
|
||||
<a href="#message_<?php echo h($post['post_id']); ?>">#<?php echo h($post['post_id'])?></a>
|
||||
</span>
|
||||
<?php
|
||||
}
|
||||
|
|
@ -76,7 +76,7 @@
|
|||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="discussionBoxTD discussionBoxTDbottom" colspan = "2">
|
||||
<td class="discussionBoxTD discussionBoxTDbottom" colspan="2">
|
||||
<table style="width:100%">
|
||||
<tr>
|
||||
<td>
|
||||
|
|
@ -90,14 +90,14 @@
|
|||
echo $this->Form->postLink('', array('controller' => 'posts', 'action' => 'delete', h($post['id']), h($context)), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete this post?'));
|
||||
} else {
|
||||
?>
|
||||
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title = "<?php echo __('Reply');?>" aria-label = "<?php echo __('Reply');?>"></a>
|
||||
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title="<?php echo __('Reply');?>" aria-label="<?php echo __('Reply');?>"></a>
|
||||
<?php
|
||||
}
|
||||
} else {
|
||||
echo $this->Html->link('', array('controller' => 'posts', 'action' => 'edit', h($post['id']), h($context)), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));
|
||||
echo $this->Form->postLink('', array('controller' => 'posts', 'action' => 'delete', h($post['id']), h($context)), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete this post?'));
|
||||
?>
|
||||
<a href = "<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title = "<?php echo __('Reply');?>" aria-label = "<?php echo __('Reply');?>"></a>
|
||||
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title="<?php echo __('Reply');?>" aria-label="<?php echo __('Reply');?>"></a>
|
||||
<?php
|
||||
|
||||
}
|
||||
|
|
@ -108,7 +108,7 @@
|
|||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<br />
|
||||
<br>
|
||||
<?php
|
||||
}
|
||||
?>
|
||||
|
|
@ -139,11 +139,11 @@
|
|||
?>
|
||||
<fieldset>
|
||||
<div class="input clear">
|
||||
<button type="button" title="<?php echo __('Insert a quote - just paste your quote between the [quote][/quote] tags.');?>" class="toggle-left btn btn-inverse qet" id = "quote" onclick="insertQuote()"><?php echo __('Quote');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a link to an event - just enter the event ID between the [event][/event] tags.');?>" class="toggle btn btn-inverse qet" id = "event" onclick="insertEvent()"><?php echo __('Event');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a link to a discussion thread - enter the thread\'s ID between the [thread][/thread] tags.');?>" class="toggle btn btn-inverse qet" id = "thread" onclick="insertThread()"><?php echo __('Thread');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a link [link][/link] tags.');?>" class="toggle btn btn-inverse qet" id = "link" onclick="insertLink()"><?php echo __('Link');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a code [code][/code] tags.');?>" class="toggle-right btn btn-inverse qet" id = "code" onclick="insertCode()"><?php echo __('Code');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a quote - just paste your quote between the [quote][/quote] tags.');?>" class="toggle-left btn btn-inverse qet" id="quote" onclick="insertQuote()"><?php echo __('Quote');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a link to an event - just enter the event ID between the [event][/event] tags.');?>" class="toggle btn btn-inverse qet" id="event" onclick="insertEvent()"><?php echo __('Event');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a link to a discussion thread - enter the thread\'s ID between the [thread][/thread] tags.');?>" class="toggle btn btn-inverse qet" id="thread" onclick="insertThread()"><?php echo __('Thread');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a link [link][/link] tags.');?>" class="toggle btn btn-inverse qet" id="link" onclick="insertLink()"><?php echo __('Link');?></button>
|
||||
<button type="button" title="<?php echo __('Insert a code [code][/code] tags.');?>" class="toggle-right btn btn-inverse qet" id="code" onclick="insertCode()"><?php echo __('Code');?></button>
|
||||
</div>
|
||||
<?php
|
||||
echo $this->Form->input('message', array(
|
||||
|
|
@ -154,7 +154,7 @@
|
|||
));
|
||||
?>
|
||||
</fieldset>
|
||||
<button class="btn btn-primary" onClick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send');?></button>
|
||||
<button class="btn btn-primary" onclick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send comment');?></button>
|
||||
<?php
|
||||
echo $this->Form->end();
|
||||
?>
|
||||
|
|
@ -177,7 +177,7 @@
|
|||
document.getElementById("PostMessage").value+="[Code][/Code]";
|
||||
}
|
||||
<?php if (isset($post_id) && $post_id): ?>
|
||||
$(document).ready(function() {
|
||||
$(function() {
|
||||
location.hash = "#message_<?php echo h($post_id); ?>";
|
||||
});
|
||||
<?php endif; ?>
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@
|
|||
);
|
||||
$popoverHTML = '';
|
||||
foreach ($popover as $key => $popoverElement) {
|
||||
$popoverHTML .= '<span class=\'bold\'>' . $key . '</span>: <span class=\'blue bold\'>' . $popoverElement . '</span><br />';
|
||||
$popoverHTML .= '<span class=\'bold\'>' . $key . '</span>: <span class=\'blue bold\'>' . h($popoverElement) . '</span><br />';
|
||||
}
|
||||
?>
|
||||
<a href="<?php echo $baseurl; ?>/events/view/<?php echo h($relation['Event']['id']);?>" data-toggle="popover" title="Attribute details" data-content="<?php echo h($popoverHTML); ?>" data-trigger="hover"><?php echo h($relation['Event']['id']);?></a>
|
||||
|
|
|
|||
|
|
@ -3,35 +3,33 @@ App::uses('AppHelper', 'View/Helper');
|
|||
|
||||
// Helper to retrieve org images with the given parameters
|
||||
class OrgImgHelper extends AppHelper {
|
||||
const IMG_PATH = APP . WEBROOT_DIR . DS . 'img' . DS . 'orgs' . DS;
|
||||
|
||||
public function getOrgImg($options, $returnData = false, $raw = false) {
|
||||
$imgPath = APP . WEBROOT_DIR . DS . 'img' . DS . 'orgs' . DS;
|
||||
$imgOptions = array();
|
||||
$possibleFields = array('id', 'name');
|
||||
$size = !empty($options['size']) ? $options['size'] : 48;
|
||||
foreach ($possibleFields as $field) {
|
||||
if (isset($options[$field]) && file_exists($imgPath . $options[$field] . '.png')) {
|
||||
if (isset($options[$field]) && file_exists(self::IMG_PATH . $options[$field] . '.png')) {
|
||||
$imgOptions[$field] = $options[$field] . '.png';
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!empty($imgOptions)) {
|
||||
foreach ($imgOptions as $field => $imgOption) {
|
||||
if ($raw) {
|
||||
$result = sprintf(
|
||||
'<img src="/img/orgs/%s" title="%s" width="%s" height="%s">',
|
||||
$imgOption,
|
||||
isset($options['name']) ? h($options['name']) : h($options['id']),
|
||||
(int)$size,
|
||||
(int)$size
|
||||
);
|
||||
|
||||
if (!$raw) {
|
||||
$result = sprintf(
|
||||
'<img src="/img/orgs/%s" title = "%s" style = "width: %spx; height: %spx;"/>',
|
||||
$imgOption,
|
||||
isset($options['name']) ? h($options['name']) : h($options['id']),
|
||||
h($size),
|
||||
h($size)
|
||||
);
|
||||
} else {
|
||||
$result = sprintf(
|
||||
'<a href="/organisations/view/%s"><img src="/img/orgs/%s" title = "%s" style = "width: %spx; height: %spx;"/></a>',
|
||||
'<a href="/organisations/view/%s">%s</a>',
|
||||
(empty($options['id']) ? h($options['name']) : h($options['id'])),
|
||||
$imgOption,
|
||||
isset($options['name']) ? h($options['name']) : h($options['id']),
|
||||
h($size),
|
||||
h($size)
|
||||
$result
|
||||
);
|
||||
}
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
<legend><?php echo __('ZeroMQ Server Status');?></legend>
|
||||
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
|
||||
<?php if (isset($time)): ?>
|
||||
<p><b><?php echo __('Start time');?></b>: <?php echo h($time); ?><br />
|
||||
<b><?php echo __('Settings read at');?></b>: <?php echo h($time2); ?><br />
|
||||
<b><?php echo __('Events processed');?></b>: <?php echo h($events); ?></p>
|
||||
<p><b><?php echo __('Reply time');?></b>: <?php echo h($time); ?><br>
|
||||
<b><?php echo __('Start time');?></b>: <?php echo h($time2); ?><br>
|
||||
<b><?php echo __('Events processed');?></b>: <?php echo h($events); ?><br>
|
||||
<b><?php echo __('Messages processed');?></b>: <?php echo h($messages); ?></p>
|
||||
<?php else: ?>
|
||||
<p><?php echo __('The ZeroMQ server is unreachable.');?></p>
|
||||
<?php endif; ?>
|
||||
|
|
|
|||
|
|
@ -907,7 +907,7 @@
|
|||
"id": "46",
|
||||
"name": "All current domains belonging to known malicious DGAs",
|
||||
"provider": "osint.bambenekconsulting.com",
|
||||
"url": "http://osint.bambenekconsulting.com/feeds/dga-feed-high.csv",
|
||||
"url": "https://osint.bambenekconsulting.com/feeds/dga-feed-high.csv",
|
||||
"rules": "",
|
||||
"enabled": true,
|
||||
"distribution": "3",
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit cb9bff18e8c94584588ec1333affb4959cdc08d0
|
||||
Subproject commit 881a5664acfbb0e3ef68c70b641a966859ed2f4f
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 597a6878f32b7416b0b910e8f2c4ae2cef086113
|
||||
|
|
@ -1,15 +1,20 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import zmq
|
||||
from zmq.auth.thread import ThreadAuthenticator
|
||||
from zmq.utils.monitor import recv_monitor_message
|
||||
import sys
|
||||
import redis
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import threading
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(name)s:%(message)s")
|
||||
|
||||
|
||||
def check_pid(pid):
|
||||
""" Check For the existence of a unix pid. """
|
||||
if not pid:
|
||||
|
|
@ -24,102 +29,185 @@ def check_pid(pid):
|
|||
return True
|
||||
|
||||
|
||||
class MISPZMQ:
|
||||
EVENT_MAP = {}
|
||||
for name in dir(zmq):
|
||||
if name.startswith("EVENT_"):
|
||||
value = getattr(zmq, name)
|
||||
EVENT_MAP[value] = name
|
||||
|
||||
|
||||
def event_monitor(monitor, logger):
|
||||
while monitor.poll():
|
||||
evt = recv_monitor_message(monitor)
|
||||
if evt["event"] == zmq.EVENT_MONITOR_STOPPED:
|
||||
break
|
||||
evt.update({"description": EVENT_MAP[evt["event"]]})
|
||||
logger.debug("ZMQ event: {}".format(evt))
|
||||
monitor.close()
|
||||
|
||||
|
||||
class MispZmq:
|
||||
message_count = 0
|
||||
publish_count = 0
|
||||
|
||||
monitor_thread = None
|
||||
auth = None
|
||||
socket = None
|
||||
pidfile = None
|
||||
|
||||
def __init__(self):
|
||||
self.current_location = Path(__file__).parent
|
||||
self.pidfile = self.current_location / "mispzmq.pid"
|
||||
self.publishCount = 0
|
||||
self._logger = logging.getLogger()
|
||||
|
||||
self.tmp_location = Path(__file__).parent.parent / "tmp"
|
||||
self.pidfile = self.tmp_location / "mispzmq.pid"
|
||||
if self.pidfile.exists():
|
||||
with open(self.pidfile.as_posix()) as f:
|
||||
pid = f.read()
|
||||
if check_pid(pid):
|
||||
raise Exception('mispzmq already running on PID {}'.format(pid))
|
||||
raise Exception("mispzmq already running on PID {}".format(pid))
|
||||
else:
|
||||
# Cleanup
|
||||
self.pidfile.unlink()
|
||||
if (self.current_location / 'settings.json').exists():
|
||||
self.setup()
|
||||
if (self.tmp_location / "mispzmq_settings.json").exists():
|
||||
self._setup()
|
||||
else:
|
||||
raise Exception("The settings file is missing.")
|
||||
|
||||
def setup(self):
|
||||
with open((self.current_location / 'settings.json').as_posix()) as settings_file:
|
||||
def _setup(self):
|
||||
with open((self.tmp_location / "mispzmq_settings.json").as_posix()) as settings_file:
|
||||
self.settings = json.load(settings_file)
|
||||
self.namespace = self.settings["redis_namespace"]
|
||||
self.r = redis.StrictRedis(host=self.settings["redis_host"], db=self.settings["redis_database"],
|
||||
password=self.settings["redis_password"], port=self.settings["redis_port"],
|
||||
decode_responses=True)
|
||||
self.timestampSettings = time.time()
|
||||
self.timestamp_settings = time.time()
|
||||
self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
|
||||
self.settings["redis_database"]))
|
||||
|
||||
def handleCommand(self, command):
|
||||
def _setup_zmq(self):
|
||||
context = zmq.Context()
|
||||
|
||||
if "username" in self.settings and self.settings["username"]:
|
||||
if "password" not in self.settings or not self.settings["password"]:
|
||||
raise Exception("When username is set, password cannot be empty.")
|
||||
|
||||
self.auth = ThreadAuthenticator(context)
|
||||
self.auth.start()
|
||||
self.auth.configure_plain(domain="*", passwords={self.settings["username"]: self.settings["password"]})
|
||||
else:
|
||||
if self.auth:
|
||||
self.auth.stop()
|
||||
self.auth = None
|
||||
|
||||
self.socket = context.socket(zmq.PUB)
|
||||
if self.settings["username"]:
|
||||
self.socket.plain_server = True # must come before bind
|
||||
self.socket.bind("tcp://*:{}".format(self.settings["port"]))
|
||||
self._logger.debug("ZMQ listening on tcp://*:{}".format(self.settings["port"]))
|
||||
|
||||
if self._logger.isEnabledFor(logging.DEBUG):
|
||||
monitor = self.socket.get_monitor_socket()
|
||||
self.monitor_thread = threading.Thread(target=event_monitor, args=(monitor, self._logger))
|
||||
self.monitor_thread.start()
|
||||
else:
|
||||
if self.monitor_thread:
|
||||
self.socket.disable_monitor()
|
||||
self.monitor_thread = None
|
||||
|
||||
def _handle_command(self, command):
|
||||
if command == "kill":
|
||||
print("Kill command received, shutting down.")
|
||||
self.pidfile.unlink()
|
||||
self._logger.info("Kill command received, shutting down.")
|
||||
self.clean()
|
||||
sys.exit()
|
||||
|
||||
elif command == "reload":
|
||||
print("Reload command received, reloading settings from file.")
|
||||
self.setup()
|
||||
self._logger.info("Reload command received, reloading settings from file.")
|
||||
self._setup()
|
||||
self._setup_zmq()
|
||||
|
||||
elif command == "status":
|
||||
print("Status command received, responding with latest stats.")
|
||||
self._logger.info("Status command received, responding with latest stats.")
|
||||
self.r.delete("{}:status".format(self.namespace))
|
||||
self.r.lpush("{}:status".format(self.namespace),
|
||||
json.dumps({"timestamp": time.time(),
|
||||
"timestampSettings": self.timestampSettings,
|
||||
"publishCount": self.publishCount}))
|
||||
"timestampSettings": self.timestamp_settings,
|
||||
"publishCount": self.publish_count,
|
||||
"messageCount": self.message_count}))
|
||||
else:
|
||||
self._logger.warning("Received invalid command '{}'.".format(command))
|
||||
|
||||
def createPidFile(self):
|
||||
with open(self.pidfile.as_posix(), 'w') as f:
|
||||
def _create_pid_file(self):
|
||||
with open(self.pidfile.as_posix(), "w") as f:
|
||||
f.write(str(os.getpid()))
|
||||
|
||||
def pubMessage(self, topic, data, socket):
|
||||
socket.send_string("{} {}".format(topic, data))
|
||||
if topic is 'misp_json':
|
||||
self.publishCount += 1
|
||||
def _pub_message(self, topic, data):
|
||||
self.socket.send_string("{} {}".format(topic, data))
|
||||
|
||||
def clean(self):
|
||||
if self.monitor_thread:
|
||||
self.socket.disable_monitor()
|
||||
if self.auth:
|
||||
self.auth.stop()
|
||||
if self.socket:
|
||||
self.socket.close()
|
||||
if self.pidfile:
|
||||
self.pidfile.unlink()
|
||||
|
||||
def main(self):
|
||||
start_time = int(time.time())
|
||||
self.createPidFile()
|
||||
self._create_pid_file()
|
||||
self._setup_zmq()
|
||||
time.sleep(1)
|
||||
|
||||
status_array = [
|
||||
"And when you're dead I will be still alive.",
|
||||
"And believe me I am still alive.",
|
||||
"I'm doing science and I'm still alive.",
|
||||
"I feel FANTASTIC and I'm still alive.",
|
||||
"While you're dying I'll be still alive."
|
||||
"While you're dying I'll be still alive.",
|
||||
]
|
||||
topics = ["misp_json", "misp_json_event", "misp_json_attribute", "misp_json_sighting",
|
||||
"misp_json_organisation", "misp_json_user", "misp_json_conversation",
|
||||
"misp_json_object", "misp_json_object_reference", "misp_json_audit",
|
||||
"misp_json_tag"
|
||||
"misp_json_tag",
|
||||
]
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.PUB)
|
||||
socket.bind("tcp://*:{}".format(self.settings["port"]))
|
||||
time.sleep(1)
|
||||
|
||||
lists = ["{}:command".format(self.namespace)]
|
||||
for topic in topics:
|
||||
lists.append("{}:data:{}".format(self.namespace, topic))
|
||||
|
||||
while True:
|
||||
command = self.r.lpop("{}:command".format(self.namespace))
|
||||
if command is not None:
|
||||
self.handleCommand(command)
|
||||
message_received = False
|
||||
for topic in topics:
|
||||
data = self.r.lpop("{}:data:{}".format(self.namespace, topic))
|
||||
if data is not None:
|
||||
self.pubMessage(topic, data, socket)
|
||||
message_received = True
|
||||
if not message_received:
|
||||
time.sleep(0.1)
|
||||
current_time = 10 * time.time()
|
||||
temp_start_time = 10 * start_time
|
||||
time_delta = int(current_time - temp_start_time)
|
||||
if time_delta % 100 == 0:
|
||||
status_entry = int(time_delta / 100 % 5)
|
||||
data = self.r.blpop(lists, timeout=10)
|
||||
|
||||
if data is None:
|
||||
# redis timeout expired
|
||||
current_time = int(time.time())
|
||||
time_delta = current_time - int(self.timestamp_settings)
|
||||
status_entry = int(time_delta / 10 % 5)
|
||||
status_message = {
|
||||
'status': status_array[status_entry],
|
||||
'uptime': int(time.time()) - start_time
|
||||
"status": status_array[status_entry],
|
||||
"uptime": current_time - int(self.timestamp_settings)
|
||||
}
|
||||
self.pubMessage('misp_json_self', json.dumps(status_message), socket)
|
||||
self._pub_message("misp_json_self", json.dumps(status_message))
|
||||
self._logger.debug("No message received for 10 seconds, sending ZMQ status message.")
|
||||
else:
|
||||
key, value = data
|
||||
key = key.replace("{}:".format(self.namespace), "")
|
||||
if key == "command":
|
||||
self._handle_command(value)
|
||||
elif key.startswith("data:"):
|
||||
topic = key.split(":")[1]
|
||||
self._logger.debug("Received data for topic '{}', sending to ZMQ.".format(topic))
|
||||
self._pub_message(topic, value)
|
||||
self.message_count += 1
|
||||
if topic == "misp_json":
|
||||
self.publish_count += 1
|
||||
else:
|
||||
self._logger.warning("Received invalid message '{}'.".format(key))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mzq = MISPZMQ()
|
||||
mzq.main()
|
||||
mzq = MispZmq()
|
||||
try:
|
||||
mzq.main()
|
||||
except KeyboardInterrupt:
|
||||
mzq.clean()
|
||||
|
|
|
|||
|
|
@ -618,7 +618,7 @@ class StixBuilder():
|
|||
@CustomObject(custom_object_type, [
|
||||
('id', StringProperty(required=True)),
|
||||
('labels', ListProperty(labels, required=True)),
|
||||
('x_misp_value', StringProperty(required=True)),
|
||||
('x_misp_values', DictionaryProperty(required=True)),
|
||||
('created', TimestampProperty(required=True, precision='millisecond')),
|
||||
('modified', TimestampProperty(required=True, precision='millisecond')),
|
||||
('created_by_ref', StringProperty(required=True)),
|
||||
|
|
|
|||
|
|
@ -793,7 +793,7 @@ class StixParser():
|
|||
@staticmethod
|
||||
def fetch_uuid(object_id):
|
||||
try:
|
||||
return "-".join(object_id.split("-")[-5:])
|
||||
return uuid.UUID('-'.join(object_id.split("-")[1:]))
|
||||
except Exception:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
|
@ -841,7 +841,7 @@ class StixFromMISPParser(StixParser):
|
|||
for reference in self.event.information_source.references:
|
||||
self.misp_event.add_attribute(**{'type': 'link', 'value': reference})
|
||||
if package.ttps:
|
||||
for ttp in package.ttps.ttps:
|
||||
for ttp in package.ttps.ttp:
|
||||
ttp_id = '-'.join((part for part in ttp.id_.split('-')[-5:]))
|
||||
ttp_type = 'galaxy' if ttp_id in self.galaxies_references else 'object'
|
||||
self.parse_ttp(ttp, ttp_type, ttp_id)
|
||||
|
|
@ -1205,7 +1205,7 @@ class ExternalStixParser(StixParser):
|
|||
if self.event.observables:
|
||||
self.parse_external_observable(self.event.observables.observables)
|
||||
if self.event.ttps:
|
||||
self.parse_ttps(self.event.ttps.ttps)
|
||||
self.parse_ttps(self.event.ttps.ttp)
|
||||
if self.event.courses_of_action:
|
||||
self.parse_coa(self.event.courses_of_action)
|
||||
if self.dns_objects:
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 8c4e2a8e8b4d3ec287c43dcae129123fcde8cb2f
|
||||
Subproject commit 43ee8e949295759240b14e4905c45b5d6b6336e0
|
||||
|
|
@ -2113,7 +2113,7 @@ function quickFilter(passedArgs, url) {
|
|||
var passedArgs = [];
|
||||
}
|
||||
if( $('#quickFilterField').val().trim().length > 0){
|
||||
passedArgs["searchall"] = $('#quickFilterField').val().trim();
|
||||
passedArgs["searchall"] = encodeURIComponent($('#quickFilterField').val().trim());
|
||||
for (var key in passedArgs) {
|
||||
if (key !== 'page') {
|
||||
url += "/" + key + ":" + passedArgs[key];
|
||||
|
|
@ -2148,7 +2148,7 @@ function runIndexQuickFilter(preserveParams) {
|
|||
searchKey = $('#quickFilterField').data('searchkey');
|
||||
}
|
||||
if ( $('#quickFilterField').val().trim().length > 0){
|
||||
passedArgsArray[searchKey] = $('#quickFilterField').val().trim();
|
||||
passedArgsArray[searchKey] = encodeURIComponent($('#quickFilterField').val().trim());
|
||||
}
|
||||
url = here;
|
||||
if (typeof preserveParams !== "undefined") {
|
||||
|
|
@ -2168,7 +2168,7 @@ function executeFilter(passedArgs, url) {
|
|||
}
|
||||
|
||||
function quickFilterTaxonomy(taxonomy_id, passedArgs) {
|
||||
var url = "/taxonomies/view/" + taxonomy_id + "/filter:" + $('#quickFilterField').val();
|
||||
var url = "/taxonomies/view/" + taxonomy_id + "/filter:" + encodeURIComponent($('#quickFilterField').val());
|
||||
window.location.href=url;
|
||||
}
|
||||
|
||||
|
|
@ -2176,7 +2176,7 @@ function quickFilterRemoteEvents(passedArgs, id) {
|
|||
passedArgs["searchall"] = $('#quickFilterField').val();
|
||||
var url = "/servers/previewIndex/" + id;
|
||||
for (var key in passedArgs) {
|
||||
url += "/" + key + ":" + passedArgs[key];
|
||||
url += "/" + key + ":" + encodeURIComponent(passedArgs[key]);
|
||||
}
|
||||
window.location.href=url;
|
||||
}
|
||||
|
|
@ -2297,28 +2297,28 @@ function indexAddRule(param) {
|
|||
var found = false;
|
||||
if (filterContext == 'event') {
|
||||
if (param.data.param1 == "date") {
|
||||
var val1 = escape($('#EventSearch' + param.data.param1 + 'from').val());
|
||||
var val2 = escape($('#EventSearch' + param.data.param1 + 'until').val());
|
||||
var val1 = encodeURIComponent($('#EventSearch' + param.data.param1 + 'from').val());
|
||||
var val2 = encodeURIComponent($('#EventSearch' + param.data.param1 + 'until').val());
|
||||
if (val1 != "") filtering.date.from = val1;
|
||||
if (val2 != "") filtering.date.until = val2;
|
||||
} else if (param.data.param1 == "published") {
|
||||
var value = escape($('#EventSearchpublished').val());
|
||||
var value = encodeURIComponent($('#EventSearchpublished').val());
|
||||
if (value != "") filtering.published = value;
|
||||
} else if (param.data.param1 == "hasproposal") {
|
||||
var value = escape($('#EventSearchhasproposal').val());
|
||||
var value = encodeURIComponent($('#EventSearchhasproposal').val());
|
||||
if (value != "") filtering.hasproposal = value;
|
||||
} else {
|
||||
var value = escape($('#EventSearch' + param.data.param1).val());
|
||||
var operator = operators[escape($('#EventSearchbool').val())];
|
||||
var value = encodeURIComponent($('#EventSearch' + param.data.param1).val());
|
||||
var operator = operators[encodeURIComponent($('#EventSearchbool').val())];
|
||||
if (value != "" && filtering[param.data.param1][operator].indexOf(value) < 0) filtering[param.data.param1][operator].push(value);
|
||||
}
|
||||
} else if (filterContext == 'user') {
|
||||
if (differentFilters.indexOf(param.data.param1) != -1) {
|
||||
var value = escape($('#UserSearch' + param.data.param1).val());
|
||||
var value = encodeURIComponent($('#UserSearch' + param.data.param1).val());
|
||||
if (value != "") filtering[param.data.param1] = value;
|
||||
} else {
|
||||
var value = escape($('#UserSearch' + param.data.param1).val());
|
||||
var operator = operators[escape($('#UserSearchbool').val())];
|
||||
var value = encodeURIComponent($('#UserSearch' + param.data.param1).val());
|
||||
var operator = operators[encodeURIComponent($('#UserSearchbool').val())];
|
||||
if (value != "" && filtering[param.data.param1][operator].indexOf(value) < 0) filtering[param.data.param1][operator].push(value);
|
||||
}
|
||||
}
|
||||
|
|
@ -2537,7 +2537,7 @@ function serverSettingSubmitForm(name, setting, id) {
|
|||
}
|
||||
|
||||
function updateOrgCreateImageField(string) {
|
||||
string = escape(string);
|
||||
string = encodeURIComponent(string);
|
||||
$.ajax({
|
||||
url:'/img/orgs/' + string + '.png',
|
||||
type:'HEAD',
|
||||
|
|
@ -3605,11 +3605,11 @@ function syncUserSelected() {
|
|||
function filterAttributes(filter, id) {
|
||||
url = "/events/viewEventAttributes/" + id;
|
||||
if(filter === 'value'){
|
||||
filter = $('#quickFilterField').val().trim();
|
||||
filter = encodeURIComponent($('#quickFilterField').val().trim());
|
||||
url += filter.length > 0 ? "/searchFor:" + filter : "";
|
||||
} else if(filter !== 'all') {
|
||||
url += "/attributeFilter:" + filter
|
||||
filter = $('#quickFilterField').val().trim();
|
||||
filter = encodeURIComponent($('#quickFilterField').val().trim());
|
||||
url += filter.length > 0 ? "/searchFor:" + filter : "";
|
||||
}
|
||||
if (deleted) url += '/deleted:true';
|
||||
|
|
|
|||
1813
db_schema.json
1813
db_schema.json
File diff suppressed because it is too large
Load Diff
4105
docs/Changelog.md
4105
docs/Changelog.md
File diff suppressed because it is too large
Load Diff
|
|
@ -1,12 +1,14 @@
|
|||
# INSTALLATION INSTRUCTIONS
|
||||
## for Kali Linux 2019.1
|
||||
## for Kali Linux 2020.2
|
||||
# 0/ Quick MISP Instance on Kali Linux - Status
|
||||
|
||||
This has been tested by @SteveClement on 20190221
|
||||
This has been tested by @SteveClement on 20200513
|
||||
|
||||
# 1/ Prepare Kali with a MISP User
|
||||
|
||||
To install MISP on Kali copy paste this in your r00t shell:
|
||||
This only works on Kali 2020.x 2019.x was year(s) ago(tm).
|
||||
|
||||
To install MISP on Kali copy paste this in your shell:
|
||||
```bash
|
||||
wget -O /tmp/misp-kali.sh https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh && bash /tmp/misp-kali.sh
|
||||
```
|
||||
|
|
|
|||
|
|
@ -243,9 +243,6 @@ installCoreRHEL () {
|
|||
cd $PATH_TO_MISP/app/files/scripts/mixbox
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd $PATH_TO_MISP/cti-python-stix2
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
|
@ -297,6 +294,25 @@ installCoreRHEL () {
|
|||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
|
||||
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
# BROKEN: This needs to be tested on RHEL/CentOS
|
||||
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# Enable dependencies detection in the diagnostics page
|
||||
# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
|
||||
# The LD_LIBRARY_PATH setting is needed for rh-git218 to work
|
||||
|
|
|
|||
|
|
@ -222,9 +222,6 @@ installCoreRHEL () {
|
|||
cd $PATH_TO_MISP/app/files/scripts/mixbox
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd $PATH_TO_MISP/cti-python-stix2
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
|
@ -277,6 +274,25 @@ installCoreRHEL () {
|
|||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
|
||||
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
# BROKEN: This needs to be tested on RHEL/CentOS
|
||||
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# Enable python3 for php-fpm
|
||||
sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/php-fpm.d/www.conf
|
||||
sudo systemctl restart php-fpm.service
|
||||
|
|
@ -306,10 +322,10 @@ installCake_RHEL ()
|
|||
sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer
|
||||
cd $PATH_TO_MISP/app
|
||||
# Update composer.phar (optional)
|
||||
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
|
||||
$SUDO_WWW php composer-setup.php
|
||||
$SUDO_WWW php -r "unlink('composer-setup.php');"
|
||||
#$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
#$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
|
||||
#$SUDO_WWW php composer-setup.php
|
||||
#$SUDO_WWW php -r "unlink('composer-setup.php');"
|
||||
$SUDO_WWW php composer.phar install
|
||||
|
||||
## sudo yum install php-redis -y
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ bash /tmp/INSTALL.sh -c
|
|||
aptUpgrade () {
|
||||
debug "Upgrading system"
|
||||
checkAptLock
|
||||
sudo apt-get update
|
||||
|
||||
# If we run in non-interactive mode, make sure we do not stop all of a sudden
|
||||
if [[ "${PACKER}" == "1" || "${UNATTENDED}" == "1" ]]; then
|
||||
|
|
@ -110,7 +109,7 @@ installDepsPhp72 () {
|
|||
debug "Installing PHP 7.2 dependencies"
|
||||
PHP_ETC_BASE=/etc/php/7.2
|
||||
PHP_INI=${PHP_ETC_BASE}/apache2/php.ini
|
||||
sudo apt update
|
||||
checkAptLock
|
||||
sudo apt install -qy \
|
||||
libapache2-mod-php \
|
||||
php php-cli \
|
||||
|
|
@ -167,8 +166,6 @@ installCore () {
|
|||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-maec
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd ${PATH_TO_MISP}/cti-python-stix2
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
|
@ -176,6 +173,23 @@ installCore () {
|
|||
# install PyMISP
|
||||
cd ${PATH_TO_MISP}/PyMISP
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# install pydeep
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ bash /tmp/INSTALL.sh -c
|
|||
aptUpgrade () {
|
||||
debug "Upgrading system"
|
||||
checkAptLock
|
||||
sudo apt-get update
|
||||
|
||||
# If we run in non-interactive mode, make sure we do not stop all of a sudden
|
||||
if [[ "${PACKER}" == "1" || "${UNATTENDED}" == "1" ]]; then
|
||||
|
|
@ -110,7 +109,7 @@ installDepsPhp74 () {
|
|||
debug "Installing PHP 7.4 dependencies"
|
||||
PHP_ETC_BASE=/etc/php/7.4
|
||||
PHP_INI=${PHP_ETC_BASE}/apache2/php.ini
|
||||
sudo apt update
|
||||
checkAptLock
|
||||
sudo apt install -qy \
|
||||
libapache2-mod-php \
|
||||
php php-cli \
|
||||
|
|
@ -167,8 +166,6 @@ installCore () {
|
|||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-maec
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd ${PATH_TO_MISP}/cti-python-stix2
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
|
@ -176,6 +173,23 @@ installCore () {
|
|||
# install PyMISP
|
||||
cd ${PATH_TO_MISP}/PyMISP
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# install pydeep
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ mispmodules () {
|
|||
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
|
||||
# If you build an egg, the user you build it as need write permissions in the CWD
|
||||
sudo chgrp $WWW_USER .
|
||||
sudo chmod g+w .
|
||||
sudo chmod og+w .
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
|
||||
sudo chgrp staff .
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ checkFlavour () {
|
|||
FLAVOUR="$(. /etc/os-release && echo "$ID"| tr '[:upper:]' '[:lower:]')"
|
||||
fi
|
||||
|
||||
case "$FLAVOUR" in
|
||||
case "${FLAVOUR}" in
|
||||
ubuntu)
|
||||
if command_exists lsb_release; then
|
||||
dist_version="$(lsb_release --codename | cut -f2)"
|
||||
|
|
@ -139,7 +139,7 @@ checkFlavour () {
|
|||
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
|
||||
dist_version=${dist_version:0:1}
|
||||
fi
|
||||
echo "$FLAVOUR support is experimental at the moment"
|
||||
echo "${FLAVOUR} support is experimental at the moment"
|
||||
;;
|
||||
rhel|ol|sles)
|
||||
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
|
||||
|
|
@ -147,7 +147,7 @@ checkFlavour () {
|
|||
dist_version=${dist_version:0:1} # Only interested about major version
|
||||
fi
|
||||
# Only tested for RHEL 7 so far
|
||||
echo "$FLAVOUR support is experimental at the moment"
|
||||
echo "${FLAVOUR} support is experimental at the moment"
|
||||
;;
|
||||
*)
|
||||
if command_exists lsb_release; then
|
||||
|
|
@ -160,7 +160,7 @@ checkFlavour () {
|
|||
esac
|
||||
|
||||
# FIXME: The below want to be refactored
|
||||
if [ "$FLAVOUR" == "ubuntu" ]; then
|
||||
if [ "${FLAVOUR}" == "ubuntu" ]; then
|
||||
RELEASE=$(lsb_release -s -r)
|
||||
debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}"
|
||||
else
|
||||
|
|
@ -183,7 +183,7 @@ check_forked () {
|
|||
if [ "$lsb_release_exit_code" = "0" ]; then
|
||||
# Print info about current distro
|
||||
cat <<-EOF
|
||||
You're using '$FLAVOUR' version '$dist_version'.
|
||||
You're using '${FLAVOUR}' version '${dist_version}'.
|
||||
EOF
|
||||
# Get the upstream release info
|
||||
FLAVOUR=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
|
||||
|
|
@ -191,10 +191,10 @@ EOF
|
|||
|
||||
# Print info about upstream distro
|
||||
cat <<-EOF
|
||||
Upstream release is '$FLAVOUR' version '$dist_version'.
|
||||
Upstream release is '${FLAVOUR}' version '$dist_version'.
|
||||
EOF
|
||||
else
|
||||
if [ -r /etc/debian_version ] && [ "$FLAVOUR" != "ubuntu" ] && [ "$FLAVOUR" != "raspbian" ]; then
|
||||
if [[ -r /etc/debian_version ]] && [[ "${FLAVOUR}" != "ubuntu" ]] && [[ "${FLAVOUR}" != "raspbian" ]]; then
|
||||
# We're Debian and don't even know it!
|
||||
FLAVOUR=debian
|
||||
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
|
||||
|
|
@ -216,7 +216,7 @@ EOF
|
|||
|
||||
checkInstaller () {
|
||||
# Workaround: shasum is not available on RHEL, only checking sha512
|
||||
if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then
|
||||
if [[ "${FLAVOUR}" == "rhel" ]] || [[ "${FLAVOUR}" == "centos" ]]; then
|
||||
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
|
||||
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
|
||||
chsum=$(cat /tmp/INSTALL.sh.sha512)
|
||||
|
|
@ -228,8 +228,8 @@ checkInstaller () {
|
|||
fi
|
||||
else
|
||||
# TODO: Implement $FLAVOUR checks and install depending on the platform we are on
|
||||
if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then
|
||||
sudo apt update
|
||||
if [[ $(which shasum > /dev/null 2>&1 ; echo $?) -ne 0 ]]; then
|
||||
checkAptLock
|
||||
sudo apt install libdigest-sha-perl -qyy
|
||||
fi
|
||||
# SHAsums to be computed, not the -- notatiation is for ease of use with rhash
|
||||
|
|
@ -252,12 +252,12 @@ checkInstaller () {
|
|||
|
||||
# Extract manufacturer
|
||||
checkManufacturer () {
|
||||
if [ -z $(which dmidecode) ]; then
|
||||
if [[ -z $(which dmidecode) ]]; then
|
||||
checkAptLock
|
||||
sudo apt install dmidecode -qy
|
||||
fi
|
||||
MANUFACTURER=$(sudo dmidecode -s system-manufacturer)
|
||||
echo $MANUFACTURER
|
||||
debug ${MANUFACTURER}
|
||||
}
|
||||
|
||||
# Dynamic horizontal spacer if needed, for autonomeous an no progress bar install, we are static.
|
||||
|
|
@ -319,7 +319,7 @@ progress () {
|
|||
checkLocale () {
|
||||
debug "Checking Locale"
|
||||
# If locale is missing, generate and install a common UTF-8
|
||||
if [[ ! -f /etc/default/locale || $(wc -l /etc/default/locale| cut -f 1 -d\ ) == "1" ]]; then
|
||||
if [[ ! -f /etc/default/locale || $(wc -l /etc/default/locale| cut -f 1 -d\ ) -eq "1" ]]; then
|
||||
checkAptLock
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt install locales -qy
|
||||
sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/g' /etc/locale.gen
|
||||
|
|
@ -330,6 +330,7 @@ checkLocale () {
|
|||
|
||||
# Simple function to check command exit code
|
||||
checkFail () {
|
||||
# '-ne' checks for numerical differences, '==' used for strings
|
||||
if [[ $2 -ne 0 ]]; then
|
||||
echo "iAmError: $1"
|
||||
echo "The last command exited with error code: $2"
|
||||
|
|
@ -366,7 +367,7 @@ clean () {
|
|||
# Check if misp user is present and if run as root
|
||||
checkID () {
|
||||
debug "Checking if run as root and $MISP_USER is present"
|
||||
if [[ $EUID == 0 ]]; then
|
||||
if [[ $EUID -eq 0 ]]; then
|
||||
echo "This script cannot be run as a root"
|
||||
clean > /dev/null 2>&1
|
||||
exit 1
|
||||
|
|
@ -494,14 +495,25 @@ kaliSpaceSaver () {
|
|||
echo "${RED}Not implement${NC}"
|
||||
}
|
||||
|
||||
# Because Kali is l33t we make sure we run as root
|
||||
kaliOnRootR0ckz () {
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This script must be run as root"
|
||||
exit 1
|
||||
elif [[ $(id $MISP_USER >/dev/null; echo $?) -ne 0 ]]; then
|
||||
useradd -s /bin/bash -m -G adm,cdrom,sudo,dip,plugdev,www-data,staff $MISP_USER
|
||||
echo $MISP_USER:$MISP_PASSWORD | chpasswd
|
||||
# Because Kali is l33t we make sure we DO NOT run as root
|
||||
kaliOnTheR0ckz () {
|
||||
totalRoot=$(df -k | grep /$ |awk '{ print $2 }')
|
||||
totalMem=$(cat /proc/meminfo|grep MemTotal |grep -Eo '[0-9]{1,}')
|
||||
overlay=$(df -kh |grep overlay; echo $?) # if 1 overlay NOT present
|
||||
|
||||
if [[ ${totalRoot} -lt 3059034 ]]; then
|
||||
echo "(If?) You run Kali in LiveCD mode and we need more overlay disk space."
|
||||
echo "This is defined by the total memory, you have: ${totalMem}kB which is not enough."
|
||||
echo "6-8Gb should be fine. (need >3Gb overlayFS)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${EUID} -eq 0 ]]; then
|
||||
echo "This script must NOT be run as root"
|
||||
exit 1
|
||||
elif [[ $(id ${MISP_USER} >/dev/null; echo $?) -ne 0 ]]; then
|
||||
sudo useradd -s /bin/bash -m -G adm,cdrom,sudo,dip,plugdev,www-data,staff ${MISP_USER}
|
||||
echo ${MISP_USER}:${MISP_PASSWORD} | sudo chpasswd
|
||||
else
|
||||
# TODO: Make sure we consider this further down the road
|
||||
echo "User ${MISP_USER} exists, skipping creation"
|
||||
|
|
@ -510,20 +522,25 @@ kaliOnRootR0ckz () {
|
|||
|
||||
setBaseURL () {
|
||||
debug "Setting Base URL"
|
||||
|
||||
CONN=$(ip -br -o -4 a |grep UP |head -1 |tr -d "UP")
|
||||
IFACE=`echo $CONN |awk {'print $1'}`
|
||||
IP=`echo $CONN |awk {'print $2'}| cut -f1 -d/`
|
||||
if [[ "$(checkManufacturer)" != "innotek GmbH" ]] && [[ "$(checkManufacturer)" != "VMware, Inc." ]] && [[ "$(checkManufacturer)" != "QEMU" ]]; then
|
||||
debug "We guess that this is a physical machine and cannot possibly guess what the MISP_BASEURL might be."
|
||||
if [[ "$UNATTENDED" != "1" ]]; then
|
||||
IFACE=$(echo $CONN |awk {'print $1'})
|
||||
IP=$(echo $CONN |awk {'print $2'}| cut -f1 -d/)
|
||||
|
||||
[[ -n ${MANUFACTURER} ]] || checkManufacturer
|
||||
|
||||
if [[ "${MANUFACTURER}" != "innotek GmbH" ]] && [[ "$MANUFACTURER" != "VMware, Inc." ]] && [[ "$MANUFACTURER" != "QEMU" ]]; then
|
||||
debug "We guess that this is a physical machine and cannot reliably guess what the MISP_BASEURL might be."
|
||||
|
||||
if [[ "${UNATTENDED}" != "1" ]]; then
|
||||
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
|
||||
echo "Do you want to change it now? (y/n) "
|
||||
read ANSWER
|
||||
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
|
||||
if [[ "$ANSWER" == "y" ]]; then
|
||||
if [[ ! -z $IP ]]; then
|
||||
echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI"
|
||||
echo "Thus your Base URL could be: https://$IP"
|
||||
ANSWER=$(echo ${ANSWER} |tr '[:upper:]' '[:lower:]')
|
||||
if [[ "${ANSWER}" == "y" ]]; then
|
||||
if [[ ! -z ${IP} ]]; then
|
||||
echo "It seems you have an interface called ${IFACE} UP with the following IP: ${IP} - FYI"
|
||||
echo "Thus your Base URL could be: https://${IP}"
|
||||
fi
|
||||
echo "Please enter the Base URL, e.g: 'https://example.org'"
|
||||
echo ""
|
||||
|
|
@ -537,17 +554,17 @@ setBaseURL () {
|
|||
# Webserver configuration
|
||||
FQDN='misp.local'
|
||||
fi
|
||||
elif [[ $KALI == "1" ]]; then
|
||||
elif [[ "${KALI}" == "1" ]]; then
|
||||
MISP_BASEURL="https://misp.local"
|
||||
# Webserver configuration
|
||||
FQDN='misp.local'
|
||||
elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then
|
||||
elif [[ "${MANUFACTURER}" == "innotek GmbH" ]]; then
|
||||
MISP_BASEURL='https://localhost:8443'
|
||||
IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1)
|
||||
sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443
|
||||
# Webserver configuration
|
||||
FQDN='localhost.localdomain'
|
||||
elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then
|
||||
elif [[ "${MANUFACTURER}" == "VMware, Inc." ]]; then
|
||||
MISP_BASEURL='""'
|
||||
# Webserver configuration
|
||||
FQDN='misp.local'
|
||||
|
|
@ -578,7 +595,6 @@ installRNG () {
|
|||
# Kali upgrade
|
||||
kaliUpgrade () {
|
||||
debug "Running various Kali upgrade tasks"
|
||||
sudo apt update
|
||||
checkAptLock
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt install --only-upgrade bash libc6 -y
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt autoremove -y
|
||||
|
|
@ -608,6 +624,9 @@ if [[ $(type -t checkAptLock) == "alias" ]]; then unalias checkAptLock; fi
|
|||
# Simple function to make sure APT is not locked
|
||||
checkAptLock () {
|
||||
SLEEP=3
|
||||
if [[ -n ${APT_UPDATED} ]]; then
|
||||
sudo apt update && APT_UPDATED=1
|
||||
fi
|
||||
while [ "$DONE" != "0" ]; do
|
||||
sudo apt-get check 2> /dev/null > /dev/null && DONE=0
|
||||
echo -e "${LBLUE}apt${NC} is maybe ${RED}locked${NC}, waiting ${RED}$SLEEP${NC} seconds." > /dev/tty
|
||||
|
|
@ -623,7 +642,7 @@ installDepsPhp70 () {
|
|||
debug "Installing PHP 7.0 dependencies"
|
||||
PHP_ETC_BASE=/etc/php/7.0
|
||||
PHP_INI=${PHP_ETC_BASE}/apache2/php.ini
|
||||
sudo apt update
|
||||
checkAptLock
|
||||
sudo apt install -qy \
|
||||
libapache2-mod-php \
|
||||
php php-cli \
|
||||
|
|
@ -645,15 +664,30 @@ installDepsPhp73 () {
|
|||
debug "Installing PHP 7.3 dependencies"
|
||||
PHP_ETC_BASE=/etc/php/7.3
|
||||
PHP_INI=${PHP_ETC_BASE}/apache2/php.ini
|
||||
sudo apt update
|
||||
checkAptLock
|
||||
sudo apt install -qy \
|
||||
libapache2-mod-php7.3 \
|
||||
php7.3 php7.3-cli \
|
||||
php7.3-dev \
|
||||
php7.3-json php7.3-xml php7.3-mysql php7.3-opcache php7.3-readline php7.3-mbstring \
|
||||
php-redis php-gnupg \
|
||||
php-gd
|
||||
if [[ ! -n ${KALI} ]]; then
|
||||
sudo apt install -qy \
|
||||
libapache2-mod-php7.3 \
|
||||
php7.3 php7.3-cli \
|
||||
php7.3-dev \
|
||||
php7.3-json php7.3-xml php7.3-mysql php7.3-opcache php7.3-readline php7.3-mbstring \
|
||||
php-redis php-gnupg \
|
||||
php-gd
|
||||
else
|
||||
sudo apt install -qy \
|
||||
libapache2-mod-php7.3 \
|
||||
libgpgme-dev \
|
||||
php7.3 php7.3-cli \
|
||||
php7.3-dev \
|
||||
php7.3-json php7.3-xml php7.3-mysql php7.3-opcache php7.3-readline php7.3-mbstring \
|
||||
php7.3-gd
|
||||
sudo pecl channel-update pecl.php.net
|
||||
#sudo pear config-set php_ini ${PHP_INI}
|
||||
echo "" |sudo pecl install redis
|
||||
sudo pecl install gnupg
|
||||
echo extension=gnupg.so | sudo tee ${PHP_ETC_BASE}/mods-available/gnupg.ini
|
||||
echo extension=redis.so | sudo tee ${PHP_ETC_BASE}/mods-available/redis.ini
|
||||
fi
|
||||
}
|
||||
# <snippet-end 0_installDepsPhp73.sh>
|
||||
|
||||
|
|
@ -661,7 +695,6 @@ installDepsPhp73 () {
|
|||
installDeps () {
|
||||
debug "Installing core dependencies"
|
||||
checkAptLock
|
||||
sudo apt update
|
||||
sudo apt install -qy etckeeper
|
||||
# Skip dist-upgrade for now, pulls in 500+ updated packages
|
||||
#sudo apt -y dist-upgrade
|
||||
|
|
@ -793,64 +826,56 @@ genApacheConf () {
|
|||
ServerSignature Off
|
||||
Header set X-Content-Type-Options nosniff
|
||||
Header set X-Frame-Options DENY
|
||||
</VirtualHost>" | tee /etc/apache2/sites-available/misp-ssl.conf
|
||||
</VirtualHost>" | sudo tee /etc/apache2/sites-available/misp-ssl.conf
|
||||
}
|
||||
|
||||
# Add git pull update mechanism to rc.local - TODO: Make this better
|
||||
gitPullAllRCLOCAL () {
|
||||
sed -i -e '$i \git_dirs="/usr/local/src/misp-modules/ /var/www/misp-dashboard /usr/local/src/faup /usr/local/src/mail_to_misp /usr/local/src/misp-modules /usr/local/src/viper /var/www/misp-dashboard"\n' /etc/rc.local
|
||||
sed -i -e '$i \for d in $git_dirs; do\n' /etc/rc.local
|
||||
sed -i -e '$i \ echo "Updating ${d}"\n' /etc/rc.local
|
||||
sed -i -e '$i \ cd $d && sudo git pull &\n' /etc/rc.local
|
||||
sed -i -e '$i \done\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \git_dirs="/usr/local/src/misp-modules /var/www/misp-dashboard /usr/local/src/faup /usr/local/src/mail_to_misp /usr/local/src/misp-modules /usr/local/src/viper /var/www/misp-dashboard"\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \for d in $git_dirs; do\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \ echo "Updating ${d}"\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \ cd $d && sudo git pull &\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \done\n' /etc/rc.local
|
||||
}
|
||||
|
||||
|
||||
# Main composer function
|
||||
composer () {
|
||||
sudo mkdir /var/www/.composer ; sudo chown ${WWW_USER}:${WWW_USER} /var/www/.composer
|
||||
${SUDO_WWW} sh -c "cd ${PATH_TO_MISP}/app ; php composer.phar install"
|
||||
}
|
||||
|
||||
|
||||
# TODO: FIX somehow the alias of the function does not work
|
||||
# Composer on php 7.0 does not need any special treatment the provided phar works well
|
||||
alias composer70='composer72'
|
||||
|
||||
alias composer70=composer
|
||||
# Composer on php 7.2 does not need any special treatment the provided phar works well
|
||||
composer72 () {
|
||||
cd $PATH_TO_MISP/app
|
||||
mkdir /var/www/.composer ; chown $WWW_USER:$WWW_USER /var/www/.composer
|
||||
$SUDO_WWW php composer.phar install
|
||||
}
|
||||
|
||||
# Composer on php 7.3 needs a recent version of composer.phar
|
||||
composer73 () {
|
||||
cd $PATH_TO_MISP/app
|
||||
mkdir /var/www/.composer ; chown $WWW_USER:$WWW_USER /var/www/.composer
|
||||
# Update composer.phar
|
||||
# If hash changes, check here: https://getcomposer.org/download/ and replace with the correct one
|
||||
# Current Sum for: v1.8.3
|
||||
SHA384_SUM="$(wget -q -O - https://composer.github.io/installer.sig)"
|
||||
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '$SHA384_SUM') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); exit(137); } echo PHP_EOL;"
|
||||
checkFail "composer.phar checksum failed, please investigate manually. " $?
|
||||
$SUDO_WWW php composer-setup.php
|
||||
$SUDO_WWW php -r "unlink('composer-setup.php');"
|
||||
$SUDO_WWW php composer.phar install
|
||||
}
|
||||
alias composer72=composer
|
||||
# Composer on php 7.3 does not need any special treatment the provided phar works well
|
||||
alias composer73=composer
|
||||
|
||||
# TODO: this is probably a useless function
|
||||
# Enable various core services
|
||||
enableServices () {
|
||||
update-rc.d mysql enable
|
||||
update-rc.d apache2 enable
|
||||
update-rc.d redis-server enable
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now mysql
|
||||
sudo systemctl enable --now apache2
|
||||
sudo systemctl enable --now redis-server
|
||||
}
|
||||
|
||||
# TODO: check if this makes sense
|
||||
# Generate rc.local
|
||||
genRCLOCAL () {
|
||||
if [ ! -e /etc/rc.local ]; then
|
||||
if [[ ! -e /etc/rc.local ]]; then
|
||||
echo '#!/bin/sh -e' | tee -a /etc/rc.local
|
||||
echo 'exit 0' | tee -a /etc/rc.local
|
||||
echo 'exit 0' | sudo tee -a /etc/rc.local
|
||||
chmod u+x /etc/rc.local
|
||||
fi
|
||||
|
||||
sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
|
||||
sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
|
||||
sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
|
||||
sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
|
||||
sudo sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local
|
||||
}
|
||||
|
||||
# Run PyMISP tests
|
||||
|
|
@ -859,9 +884,9 @@ runTests () {
|
|||
key = \"${AUTH_KEY}\"" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py
|
||||
sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/PyMISP/
|
||||
|
||||
sudo -H -u $WWW_USER sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin master"
|
||||
sudo -H -u $WWW_USER ${PATH_TO_MISP}/venv/bin/pip install -e $PATH_TO_MISP/PyMISP/.[fileobjects,neo,openioc,virustotal,pdfexport]
|
||||
sudo -H -u $WWW_USER sh -c "cd $PATH_TO_MISP/PyMISP && ${PATH_TO_MISP}/venv/bin/python tests/testlive_comprehensive.py"
|
||||
${SUDO_WWW} sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin master"
|
||||
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -e $PATH_TO_MISP/PyMISP/.[fileobjects,neo,openioc,virustotal,pdfexport]
|
||||
${SUDO_WWW} sh -c "cd $PATH_TO_MISP/PyMISP && ${PATH_TO_MISP}/venv/bin/python tests/testlive_comprehensive.py"
|
||||
}
|
||||
|
||||
# Nuke the install, meaning remove all MISP data but no packages, this makes testing the installer faster
|
||||
|
|
|
|||
|
|
@ -233,11 +233,11 @@ sudo -u www HOME=/tmp $PATH_TO_MISP/venv/bin/pip install plyara
|
|||
# CakePHP is included as a submodule of MISP and has been fetched earlier.
|
||||
# Install CakeResque along with its dependencies if you intend to use the built in background jobs:
|
||||
cd $PATH_TO_MISP/app
|
||||
sudo -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
sudo -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink
|
||||
('composer-setup.php'); } echo PHP_EOL;"
|
||||
sudo -u www env HOME=/tmp php composer-setup.php
|
||||
sudo -u www php -r "unlink('composer-setup.php');"
|
||||
#sudo -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
#sudo -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink
|
||||
#('composer-setup.php'); } echo PHP_EOL;"
|
||||
#sudo -u www env HOME=/tmp php composer-setup.php
|
||||
#sudo -u www php -r "unlink('composer-setup.php');"
|
||||
sudo -u www HOME=/tmp php composer.phar install
|
||||
|
||||
# To use the scheduler worker for scheduled tasks, do the following:
|
||||
|
|
|
|||
|
|
@ -383,9 +383,6 @@ doas /usr/local/virtualenvs/MISP/bin/python setup.py install
|
|||
cd /var/www/htdocs/MISP/PyMISP
|
||||
doas /usr/local/virtualenvs/MISP/bin/python setup.py install
|
||||
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
|
||||
# install support for STIX 2.0
|
||||
cd /var/www/htdocs/MISP/cti-python-stix2
|
||||
doas /usr/local/virtualenvs/MISP/bin/python setup.py install
|
||||
|
|
@ -402,11 +399,11 @@ doas /usr/local/virtualenvs/MISP/bin/pip install git+https://github.com/kbandla/
|
|||
# Install CakeResque along with its dependencies if you intend to use the built in background jobs:
|
||||
cd /var/www/htdocs/MISP/app
|
||||
doas mkdir /var/www/.composer ; doas chown www:www /var/www/.composer
|
||||
EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
|
||||
doas -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
doas -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
|
||||
doas -u www env HOME=/var/www php composer-setup.php
|
||||
doas -u www php -r "unlink('composer-setup.php');"
|
||||
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
|
||||
#doas -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
#doas -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
|
||||
#doas -u www env HOME=/var/www php composer-setup.php
|
||||
#doas -u www php -r "unlink('composer-setup.php');"
|
||||
doas -u www env HOME=/var/www php composer.phar install
|
||||
|
||||
# To use the scheduler worker for scheduled tasks, do the following:
|
||||
|
|
|
|||
|
|
@ -209,9 +209,6 @@ $SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git
|
|||
cd $PATH_TO_MISP/app/files/scripts/mixbox
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd $PATH_TO_MISP/cti-python-stix2
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
|
@ -220,6 +217,25 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
|||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
# BROKEN: This needs to be tested on RHEL/CentOS
|
||||
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# Enable python3 for php-fpm
|
||||
echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php70/sysconfig/php-fpm
|
||||
sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php70/php-fpm.d/www.conf
|
||||
|
|
|
|||
|
|
@ -196,9 +196,6 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U python-magic git+https://github.
|
|||
cd $PATH_TO_MISP/app/files/scripts/mixbox
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd $PATH_TO_MISP/cti-python-stix2
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
|
@ -207,6 +204,25 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
|||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
|
||||
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
# BROKEN: This needs to be tested on RHEL/CentOS
|
||||
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# Enable dependencies detection in the diagnostics page
|
||||
# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
|
||||
echo "env[PATH] =/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf
|
||||
|
|
|
|||
|
|
@ -1,16 +1,12 @@
|
|||
# INSTALLATION INSTRUCTIONS
|
||||
## for Debian 10.2 "buster"
|
||||
## for Debian 10.3 "buster"
|
||||
|
||||
### 0/ MISP debian stable install - Status
|
||||
------------------------------------
|
||||
|
||||
!!! notice
|
||||
This is mostly the install [@SteveClement](https://twitter.com/SteveClement) uses for testing, qc and random development.
|
||||
Maintained and tested by @SteveClement on 20191122
|
||||
|
||||
!!! warning
|
||||
PHP 7.3.4-2 is not working at the moment with the packaged composer.phar<br />
|
||||
You need to manually update composer.phar as outlined below.
|
||||
Maintained and tested by @SteveClement on 20200405
|
||||
|
||||
{!generic/known-issues-debian.md!}
|
||||
|
||||
|
|
@ -64,7 +60,7 @@ sudo postfix reload
|
|||
#### Install all the dependencies (some might already be installed)
|
||||
|
||||
```bash
|
||||
sudo apt install -y \
|
||||
sudo apt install \
|
||||
curl gcc git gnupg-agent make openssl redis-server neovim zip libyara-dev \
|
||||
python3-setuptools python3-dev python3-pip python3-redis python3-zmq virtualenv \
|
||||
mariadb-client \
|
||||
|
|
@ -158,8 +154,6 @@ cd $PATH_TO_MISP/app/files/scripts/python-stix
|
|||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-maec
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd ${PATH_TO_MISP}/cti-python-stix2
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
|
@ -167,6 +161,23 @@ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
|||
# install PyMISP
|
||||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# install pydeep
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
|
||||
|
|
@ -199,10 +210,10 @@ cd $PATH_TO_MISP/app
|
|||
# Make composer cache happy
|
||||
sudo mkdir /var/www/.composer ; sudo chown $WWW_USER:$WWW_USER /var/www/.composer
|
||||
# Update composer.phar
|
||||
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
|
||||
$SUDO_WWW php composer-setup.php
|
||||
$SUDO_WWW php -r "unlink('composer-setup.php');"
|
||||
#$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
#$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
|
||||
#$SUDO_WWW php composer-setup.php
|
||||
#$SUDO_WWW php -r "unlink('composer-setup.php');"
|
||||
$SUDO_WWW php composer.phar install
|
||||
# The following is potentially not needed, but just here in case of Keyboard/Chair failures
|
||||
$SUDO_WWW php composer.phar update
|
||||
|
|
|
|||
|
|
@ -183,8 +183,6 @@ cd $PATH_TO_MISP/app/files/scripts/python-stix
|
|||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
cd $PATH_TO_MISP/app/files/scripts/python-maec
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove once stix-fixed
|
||||
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
|
||||
# install STIX2.0 library to support STIX 2.0 export:
|
||||
cd ${PATH_TO_MISP}/cti-python-stix2
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
|
|
@ -192,6 +190,23 @@ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
|||
# install PyMISP
|
||||
cd $PATH_TO_MISP/PyMISP
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
|
||||
# FIXME: Remove libfaup etc once the egg has the library baked-in
|
||||
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
|
||||
cd /tmp
|
||||
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
|
||||
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
|
||||
cd gtcaca
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
$SUDO_CMD mkdir -p build
|
||||
cd build
|
||||
$SUDO_CMD cmake .. && $SUDO_CMD make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
# install pydeep
|
||||
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ fi
|
|||
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
virtualenv -p python3 mkdocs || echo "You probably have the main Python(3) binary running exclusively somewhere, make sure it is killed."
|
||||
${PWD}/mkdocs/bin/pip install mkdocs mkdocs-material markdown-include python-markdown-comments gitchangelog
|
||||
${PWD}/mkdocs/bin/pip install mkdocs==1.0.4 mkdocs-material==4.6.3 markdown-include python-markdown-comments gitchangelog
|
||||
fi
|
||||
|
||||
# Fixing ASCII aborration introduced in: https://github.com/MISP/MISP/commit/1b028ee15a3bd2f209102cd6204e6c4bb519be97
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env python3
|
||||
'''
|
||||
Takes the MISP communities metadata [1] and generates the website page [2]
|
||||
[1] https://github.com/MISP/MISP/blob/2.4/app/files/community-metadata/defaults.json
|
||||
[2] https://www.misp-project.org/communities/
|
||||
'''
|
||||
|
||||
import json
|
||||
|
||||
code_communities_filename = '../app/files/community-metadata/defaults.json'
|
||||
website_communities_filename = '../../misp-website/_pages/communities.md'
|
||||
|
||||
|
||||
communities_header = '''
|
||||
---
|
||||
layout: page
|
||||
title: MISP Communities and MISP Feeds
|
||||
permalink: /communities/
|
||||
toc: true
|
||||
---
|
||||
|
||||
## MISP Communities
|
||||
|
||||
MISP is an open source software and it is also a large community of MISP users creating, maintaining and operating communities of users or organizations sharing information about threats or cyber security indicators worldwide. The MISP project doesn't maintain an exhaustive list of all communities relying on MISP especially that some communities use MISP internally or privately.
|
||||
|
||||
# Known Existing and Public MISP Communities
|
||||
|
||||
Each community might have specific rules to join them. Below is a brief overview of existing communities, feel free to contact the respective communities that fit your organization. Some of existing public communities might be interconnected and some might be in an island mode. By running MISP, these communities usually allow their members to connect using the MISP API, MISP user-interface or even to synchronize your MISP instance with their communities. If you want to add your MISP community to the list, don't hesitate to [contact us](mailto:info@misp-project.org).
|
||||
The <i class="icon far fa-check-circle" style="color:green;"></i> sign indicates the community is vetted by the MISP Project.
|
||||
|
||||
'''
|
||||
|
||||
communities_footer = '''
|
||||
### Adding your community to the list
|
||||
|
||||
You can add your community to the list by doing a Pull Request on the [Community Metadata json file](https://github.com/MISP/MISP/blob/2.4/app/files/community-metadata/defaults.json). Alternatively [contact us](mailto:info@misp-project.org?subject=new_community) by email and specify `name`, `uuid`, `rg_uuid`, `org_name`, `description`, `url`, `sector`, `nationality`, `type`, `email`, `logo`, `pgp_key`, `misp_project_vetted`, `scope_of_data_to_be_shared`.
|
||||
|
||||
|
||||
## MISP Feed Communities
|
||||
|
||||
MISP integrates a functionality called feed that allows to fetch directly MISP events from a server without prior agreement. Two OSINT feeds are included by default in MISP and can be enabled in any new installation. Providers and partners can provide easily their feeds by using the simple [PyMISP feed-generator](https://github.com/MISP/PyMISP/tree/master/examples/feed-generator). For more information, an article about "[Using open source intelligence feeds, OSINT, with MISP](https://www.vanimpe.eu/2016/03/23/using-open-source-intelligence-osint-with-misp/)".
|
||||
|
||||
### CIRCL OSINT Feed
|
||||
|
||||
[CIRCL](https://www.circl.lu/) provides a MISP OSINT feed from various sources including their own analysis.
|
||||
|
||||
MISP URL location is [https://www.circl.lu/doc/misp/feed-osint](https://www.circl.lu/doc/misp/feed-osint).
|
||||
|
||||
### Botvrij.eu OSINT feed
|
||||
|
||||
[Botvrij.eu](http://www.botvrij.eu/) provides a MISP OSINT feed out of public report.
|
||||
|
||||
MISP URL location is [http://www.botvrij.eu/data/feed-osint](http://www.botvrij.eu/data/feed-osint).
|
||||
'''
|
||||
|
||||
|
||||
vetted_image = ' <i class="icon far fa-check-circle" style="color:green;"></i>\n'
|
||||
|
||||
|
||||
with open(code_communities_filename, 'r') as f_in:
|
||||
entries = json.load(f_in)
|
||||
|
||||
|
||||
with open(website_communities_filename, 'w') as f_out:
|
||||
f_out.write(communities_header)
|
||||
for entry in entries:
|
||||
f_out.write(f'### {entry["name"]}')
|
||||
f_out.write(vetted_image if entry.get('misp_project_vetted') else '\n')
|
||||
if entry.get('logo'):
|
||||
f_out.write(f'![Logo]({entry["logo"]}){{: style="float: right; max-width: 300px; max-height: 150px;"}}\n')
|
||||
if entry.get('url'):
|
||||
f_out.write(f'- Website: {entry["url"]}\n')
|
||||
if entry.get('type'):
|
||||
f_out.write(f'- Type: {entry["type"]}\n')
|
||||
if entry.get('sector'):
|
||||
f_out.write(f'- Sector: {entry["sector"]}\n')
|
||||
if entry.get('scope'):
|
||||
f_out.write(f'- Scope of data to be shared: {entry["scope_of_data_to_be_shared"]}\n')
|
||||
if entry.get('nationality'):
|
||||
f_out.write(f'- Nationality: {entry["nationality"]}\n')
|
||||
if entry.get('description'):
|
||||
f_out.write(f'- Description: {entry["description"]}\n')
|
||||
if entry.get('org'):
|
||||
f_out.write(f'- Managed by: {entry["org"]}\n')
|
||||
if entry.get('email'):
|
||||
f_out.write(f'- Contact: {entry["email"]}')
|
||||
if entry.get('pgp_key'):
|
||||
f_out.write('\n<details><summary>GPG key</summary>\n\n```')
|
||||
f_out.write(entry['pgp_key'])
|
||||
f_out.write('```\n</details>\n')
|
||||
f_out.write('\n')
|
||||
f_out.write('\n')
|
||||
f_out.write(communities_footer)
|
||||
|
||||
|
||||
print(f"The communities file has been generated. ({website_communities_filename}).\nPlease commit this in the misp-website repository, and publish the generated website.")
|
||||
|
|
@ -36,7 +36,7 @@ my $create=0;
|
|||
my $apply=0;
|
||||
my %CATEGORIES_CREATED = ();
|
||||
|
||||
if ($#ARGV < 0) {
|
||||
sub print_help_exit {
|
||||
print "\nUsage: misp-config --user www-data
|
||||
\t --misp-path /usr/share/misp
|
||||
\t --prefix /usr/local
|
||||
|
|
@ -45,6 +45,10 @@ if ($#ARGV < 0) {
|
|||
exit;
|
||||
}
|
||||
|
||||
if ($#ARGV < 0) {
|
||||
print_help_exit();
|
||||
}
|
||||
|
||||
sub replace_inplace_content {
|
||||
my ($filename, $buftoreplace, $replacement) = @_;
|
||||
|
||||
|
|
@ -77,6 +81,9 @@ while (my ($i, $arg) = each @ARGV) {
|
|||
if ($arg eq "--apply") {
|
||||
$apply = 1;
|
||||
}
|
||||
if ($arg eq "--help") {
|
||||
print_help_exit();
|
||||
}
|
||||
}
|
||||
|
||||
if ( ! $apply && ! $create ) {
|
||||
|
|
@ -154,7 +161,7 @@ if ($create) {
|
|||
}
|
||||
$CATEGORIES_CREATED{"$category"} = 1;
|
||||
open(my $fh, ">>", "$conf_to_write");
|
||||
print $fh "$key $value\n";
|
||||
print $fh "#$key $value\n";
|
||||
close $fh;
|
||||
|
||||
# Unset those two variables to be filled again!
|
||||
|
|
@ -182,20 +189,32 @@ if ($apply) {
|
|||
$line =~ /(\S+) (.*)/;
|
||||
my $key = "";
|
||||
|
||||
my $config_el = $1;
|
||||
my $first_config_char = substr($config_el, 0, 1);
|
||||
my $is_comment = 0;
|
||||
if ($first_config_char eq "#") {
|
||||
$is_comment = 1;
|
||||
}
|
||||
|
||||
# print("config element: $config_el\n");
|
||||
|
||||
if ($category eq "Debug") {
|
||||
# The Debug category does not... have a category
|
||||
$key = $1;
|
||||
$key = $config_el;
|
||||
} elsif ($category eq "Internal") {
|
||||
# Internal means we are patching portions of the code to configure
|
||||
if ($1 eq "redis_host") {
|
||||
if ($config_el eq "redis_host") {
|
||||
replace_inplace_content("$misp_path/app/Lib/Tools/PubSubTool.php", "localhost", $2);
|
||||
replace_inplace_content("$misp_path/app/Plugin/CakeResque/Config/config.php", "localhost", $2);
|
||||
replace_inplace_content("$misp_path/app/Vendor/kamisama/php-resque-ex/lib/Resque.php", "localhost", $2);
|
||||
replace_inplace_content("$misp_path/app/Vendor/monolog/monolog/src/Monolog/Handler/RedisHandler.php", "localhost", $2);
|
||||
}
|
||||
} else {
|
||||
$key = "$category.$1";
|
||||
$key = "$category.$config_el";
|
||||
}
|
||||
|
||||
if ($cake_config) {
|
||||
if ($cake_config && !$is_comment) {
|
||||
# print("sudo -u $misp_user $misp_path/app/Console/cake admin setSetting $key $2\n");
|
||||
system("sudo -u $misp_user $misp_path/app/Console/cake admin setSetting $key $2");
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue