2012-03-15 15:06:45 +01:00
< ? php
/**
* Application model for Cake .
*
* This file is application - wide model file . You can put all
* application - wide model - related methods here .
*
* PHP 5
*
* CakePHP ( tm ) : Rapid Development Framework ( http :// cakephp . org )
2012-09-25 15:41:58 +02:00
* Copyright 2005 - 2012 , Cake Software Foundation , Inc . ( http :// cakefoundation . org )
2012-03-15 15:06:45 +01:00
*
* Licensed under The MIT License
* Redistributions of files must retain the above copyright notice .
*
2012-09-25 15:41:58 +02:00
* @ copyright Copyright 2005 - 2012 , Cake Software Foundation , Inc . ( http :// cakefoundation . org )
2012-03-15 15:06:45 +01:00
* @ link http :// cakephp . org CakePHP ( tm ) Project
* @ package app . Model
* @ since CakePHP ( tm ) v 0.2 . 9
* @ license MIT License ( http :// www . opensource . org / licenses / mit - license . php )
*/
App :: uses ( 'Model' , 'Model' );
2012-09-20 11:34:41 +02:00
App :: uses ( 'LogableBehavior' , 'Assets.models/behaviors' );
2017-07-12 15:38:34 +02:00
App :: uses ( 'BlowfishPasswordHasher' , 'Controller/Component/Auth' );
2018-11-26 09:12:01 +01:00
App :: uses ( 'RandomTool' , 'Tools' );
2018-07-19 11:48:22 +02:00
class AppModel extends Model
{
public $name ;
2020-05-07 15:50:54 +02:00
/**
* @ var PubSubTool
*/
private $loadedPubSubTool ;
2018-07-19 11:48:22 +02:00
2019-03-05 12:24:56 +01:00
public $loadedKafkaPubTool = false ;
2018-07-19 11:48:22 +02:00
public $start = 0 ;
2020-08-18 09:52:48 +02:00
public $assetCache = [];
2020-08-18 09:26:25 +02:00
2018-07-19 11:48:22 +02:00
public $inserted_ids = array ();
2019-09-20 22:15:15 +02:00
private $__redisConnection = null ;
2018-07-19 11:48:22 +02:00
private $__profiler = array ();
public $elasticSearchClient = false ;
2020-08-13 15:58:42 +02:00
/** @var AttachmentTool|null */
private $attachmentTool ;
2018-07-19 11:48:22 +02:00
public function __construct ( $id = false , $table = null , $ds = null )
{
parent :: __construct ( $id , $table , $ds );
$this -> name = get_class ( $this );
}
// deprecated, use $db_changes
// major -> minor -> hotfix -> requires_logout
public $old_db_changes = array (
2 => array (
4 => array (
18 => false , 19 => false , 20 => false , 25 => false , 27 => false ,
32 => false , 33 => true , 38 => true , 39 => true , 40 => false ,
42 => false , 44 => false , 45 => false , 49 => true , 50 => false ,
51 => false , 52 => false , 55 => true , 56 => true , 57 => true ,
58 => false , 59 => false , 60 => false , 61 => false , 62 => false ,
63 => false , 64 => false , 65 => false , 66 => false , 67 => true ,
68 => false , 69 => false , 71 => false , 72 => false , 73 => false ,
75 => false , 77 => false , 78 => false , 79 => false , 80 => false ,
81 => false , 82 => false , 83 => false , 84 => false , 85 => false ,
86 => false , 87 => false
)
)
);
public $db_changes = array (
1 => false , 2 => false , 3 => false , 4 => true , 5 => false , 6 => false ,
7 => false , 8 => false , 9 => false , 10 => false , 11 => false , 12 => false ,
2018-09-12 19:17:32 +02:00
13 => false , 14 => false , 15 => false , 18 => false , 19 => false , 20 => false ,
2019-01-01 16:37:15 +01:00
21 => false , 22 => false , 23 => false , 24 => false , 25 => false , 26 => false ,
2019-03-19 17:23:05 +01:00
27 => false , 28 => false , 29 => false , 30 => false , 31 => false , 32 => false ,
2019-09-25 11:50:54 +02:00
33 => false , 34 => false , 35 => false , 36 => false , 37 => false , 38 => false ,
2019-11-22 21:53:51 +01:00
39 => false , 40 => false , 41 => false , 42 => false , 43 => false , 44 => false ,
2020-04-17 14:17:54 +02:00
45 => false , 46 => false , 47 => false , 48 => false , 49 => false , 50 => false ,
2020-06-07 18:54:42 +02:00
51 => false , 52 => false , 53 => false , 54 => false , 55 => false ,
2018-07-19 11:48:22 +02:00
);
2019-04-26 09:45:03 +02:00
public $advanced_updates_description = array (
2019-06-13 09:16:34 +02:00
'seenOnAttributeAndObject' => array (
'title' => 'First seen/Last seen Attribute table' ,
'description' => 'Update the Attribute table to support first_seen and last_seen feature, with a microsecond resolution.' ,
'liveOff' => true , # should the instance be offline for users other than site_admin
'recommendBackup' => true , # should the update recommend backup
'exitOnError' => false , # should the update exit on error
'requirements' => 'MySQL version must be >= 5.6' , # message stating the requirements necessary for the update
2019-07-02 15:22:21 +02:00
'record' => false , # should the update success be saved in the admin_table
2019-07-01 16:27:09 +02:00
// 'preUpdate' => 'seenOnAttributeAndObjectPreUpdate', # Function to execute before the update. If it throws an error, it cancels the update
2019-06-13 09:16:34 +02:00
'url' => '/servers/updateDatabase/seenOnAttributeAndObject/' # url pointing to the funcion performing the update
),
2019-04-26 09:45:03 +02:00
);
public $actions_description = array (
'verifyGnuPGkeys' => array (
'title' => 'Verify GnuPG keys' ,
'description' => " Run a full validation of all GnuPG keys within this instance's userbase. The script will try to identify possible issues with each key and report back on the results. " ,
'url' => '/users/verifyGPG/'
),
'databaseCleanupScripts' => array (
'title' => 'Database Cleanup Scripts' ,
'description' => 'If you run into an issue with an infinite upgrade loop (when upgrading from version ~2.4.50) that ends up filling your database with upgrade script log messages, run the following script.' ,
'url' => '/logs/pruneUpdateLogs/'
2019-10-10 12:02:23 +02:00
),
'releaseUpdateLock' => array (
'title' => 'Release update lock' ,
'description' => 'If your your database is locked and is not updating, unlock it here.' ,
2019-10-14 10:49:41 +02:00
'ignore_disabled' => true ,
2019-10-10 12:02:23 +02:00
'url' => '/servers/releaseUpdateLock/'
2019-04-26 09:45:03 +02:00
)
);
2018-07-19 11:48:22 +02:00
public function afterSave ( $created , $options = array ())
{
if ( $created ) {
$this -> inserted_ids [] = $this -> getInsertID ();
}
return true ;
}
2019-10-29 09:57:25 +01:00
public function isAcceptedDatabaseError ( $errorMessage , $dataSource )
2019-10-14 12:37:19 +02:00
{
2019-10-29 09:57:25 +01:00
$isAccepted = false ;
2020-04-14 15:04:33 +02:00
if ( $dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver' ) {
2019-10-29 09:57:25 +01:00
$errorDuplicateColumn = 'SQLSTATE[42S21]: Column already exists: 1060 Duplicate column name' ;
$errorDuplicateIndex = 'SQLSTATE[42000]: Syntax error or access violation: 1061 Duplicate key name' ;
$errorDropIndex = " /SQLSTATE \ [42000 \ ]: Syntax error or access violation: 1091 Can't DROP '[ \ w]+'; check that column \ /key exists/ " ;
$isAccepted = substr ( $errorMessage , 0 , strlen ( $errorDuplicateColumn )) === $errorDuplicateColumn ||
substr ( $errorMessage , 0 , strlen ( $errorDuplicateIndex )) === $errorDuplicateIndex ||
preg_match ( $errorDropIndex , $errorMessage ) !== 0 ;
2019-10-14 12:37:19 +02:00
} elseif ( $dataSource == 'Database/Postgres' ) {
2019-10-29 09:57:25 +01:00
$errorDuplicateColumn = '/ERROR: column "[\w]+" specified more than once/' ;
$errorDuplicateIndex = '/ERROR: relation "[\w]+" already exists/' ;
$errorDropIndex = '/ERROR: index "[\w]+" does not exist/' ;
$isAccepted = preg_match ( $errorDuplicateColumn , $errorMessage ) !== 0 ||
preg_match ( $errorDuplicateIndex , $errorMessage ) !== 0 ||
preg_match ( $errorDropIndex , $errorMessage ) !== 0 ;
2019-10-14 12:37:19 +02:00
}
2019-10-29 09:57:25 +01:00
return $isAccepted ;
2019-10-14 12:37:19 +02:00
}
2018-07-19 11:48:22 +02:00
// Generic update script
// add special cases where the upgrade does more than just update the DB
// this could become useful in the future
public function updateMISP ( $command )
{
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = false ;
2018-07-19 11:48:22 +02:00
switch ( $command ) {
case '2.4.20' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> ShadowAttribute = ClassRegistry :: init ( 'ShadowAttribute' );
$this -> ShadowAttribute -> upgradeToProposalCorrelation ();
break ;
case '2.4.25' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$newFeeds = array (
array ( 'provider' => 'CIRCL' , 'name' => 'CIRCL OSINT Feed' , 'url' => 'https://www.circl.lu/doc/misp/feed-osint' , 'enabled' => 0 ),
);
$this -> __addNewFeeds ( $newFeeds );
break ;
case '2.4.27' :
$newFeeds = array (
2019-10-31 08:48:00 +01:00
array ( 'provider' => 'Botvrij.eu' , 'name' => 'The Botvrij.eu Data' , 'url' => 'https://www.botvrij.eu/data/feed-osint' , 'enabled' => 0 )
2018-07-19 11:48:22 +02:00
);
$this -> __addNewFeeds ( $newFeeds );
break ;
case '2.4.49' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> SharingGroup = ClassRegistry :: init ( 'SharingGroup' );
$this -> SharingGroup -> correctSyncedSharingGroups ();
$this -> SharingGroup -> updateRoaming ();
break ;
case '2.4.55' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( 'addSightings' );
2018-07-19 11:48:22 +02:00
break ;
case '2.4.66' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( '2.4.66' );
2018-07-19 11:48:22 +02:00
$this -> cleanCacheFiles ();
$this -> Sighting = Classregistry :: init ( 'Sighting' );
$this -> Sighting -> addUuids ();
break ;
case '2.4.67' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( '2.4.67' );
2018-07-19 11:48:22 +02:00
$this -> Sighting = Classregistry :: init ( 'Sighting' );
$this -> Sighting -> addUuids ();
$this -> Sighting -> deleteAll ( array ( 'NOT' => array ( 'Sighting.type' => array ( 0 , 1 , 2 ))));
break ;
case '2.4.71' :
$this -> OrgBlacklist = Classregistry :: init ( 'OrgBlacklist' );
$values = array (
array ( 'org_uuid' => '58d38339-7b24-4386-b4b4-4c0f950d210f' , 'org_name' => 'Setec Astrononomy' , 'comment' => 'default example' ),
array ( 'org_uuid' => '58d38326-eda8-443a-9fa8-4e12950d210f' , 'org_name' => 'Acme Finance' , 'comment' => 'default example' )
);
foreach ( $values as $value ) {
$found = $this -> OrgBlacklist -> find ( 'first' , array ( 'conditions' => array ( 'org_uuid' => $value [ 'org_uuid' ]), 'recursive' => - 1 ));
if ( empty ( $found )) {
$this -> OrgBlacklist -> create ();
$this -> OrgBlacklist -> save ( $value );
}
}
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
break ;
case '2.4.86' :
$this -> MispObject = Classregistry :: init ( 'MispObject' );
$this -> MispObject -> removeOrphanedObjects ();
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
break ;
case 5 :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> Feed = Classregistry :: init ( 'Feed' );
$this -> Feed -> setEnableFeedCachingDefaults ();
break ;
case 8 :
$this -> Server = Classregistry :: init ( 'Server' );
$this -> Server -> restartWorkers ();
break ;
case 10 :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> Role = Classregistry :: init ( 'Role' );
$this -> Role -> setPublishZmq ();
break ;
case 12 :
$this -> __forceSettings ();
break ;
2018-11-23 14:11:33 +01:00
case 23 :
$this -> __bumpReferences ();
break ;
2019-03-25 17:35:02 +01:00
case 34 :
$this -> __fixServerPullPushRules ();
break ;
2019-09-13 11:51:05 +02:00
case 38 :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2019-09-13 11:49:12 +02:00
$this -> __addServerPriority ();
break ;
2019-12-06 15:40:51 +01:00
case 46 :
2019-12-09 09:53:18 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( 'seenOnAttributeAndObject' );
2019-06-13 09:38:39 +02:00
break ;
2020-02-26 14:37:09 +01:00
case 48 :
$dbUpdateSuccess = $this -> __generateCorrelations ();
break ;
2018-07-19 11:48:22 +02:00
default :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
break ;
}
2019-10-29 09:57:25 +01:00
return $dbUpdateSuccess ;
2018-07-19 11:48:22 +02:00
}
2019-09-13 11:49:12 +02:00
private function __addServerPriority ()
{
$this -> Server = ClassRegistry :: init ( 'Server' );
$this -> Server -> reprioritise ();
return true ;
}
2018-07-19 11:48:22 +02:00
private function __addNewFeeds ( $feeds )
{
$this -> Feed = ClassRegistry :: init ( 'Feed' );
$this -> Log = ClassRegistry :: init ( 'Log' );
$feedNames = array ();
foreach ( $feeds as $feed ) {
$feedNames [] = $feed [ 'name' ];
}
$feedNames = implode ( ', ' , $feedNames );
$result = $this -> Feed -> addDefaultFeeds ( $feeds );
$this -> Log -> create ();
$entry = array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Added new default feeds.'
);
if ( $result ) {
$entry [ 'change' ] = 'Feeds added: ' . $feedNames ;
} else {
$entry [ 'change' ] = 'Tried adding new feeds but something went wrong.' ;
}
$this -> Log -> save ( $entry );
}
// SQL scripts for updates
2019-05-08 15:07:26 +02:00
public function updateDatabase ( $command )
2018-07-19 11:48:22 +02:00
{
2019-08-14 15:22:29 +02:00
$this -> Log = ClassRegistry :: init ( 'Log' );
2020-03-02 23:09:47 +01:00
2019-04-29 11:09:04 +02:00
$liveOff = false ;
$exitOnError = false ;
2019-08-14 16:45:58 +02:00
if ( isset ( $this -> advanced_updates_description [ $command ])) {
$liveOff = isset ( $this -> advanced_updates_description [ $command ][ 'liveOff' ]) ? $this -> advanced_updates_description [ $command ][ 'liveOff' ] : $liveOff ;
$exitOnError = isset ( $this -> advanced_updates_description [ $command ][ 'exitOnError' ]) ? $this -> advanced_updates_description [ $command ][ 'exitOnError' ] : $exitOnError ;
2019-04-29 11:09:04 +02:00
}
2018-07-19 11:48:22 +02:00
$dataSourceConfig = ConnectionManager :: getDataSource ( 'default' ) -> config ;
$dataSource = $dataSourceConfig [ 'datasource' ];
$sqlArray = array ();
$indexArray = array ();
$clean = true ;
switch ( $command ) {
case 'extendServerOrganizationLength' :
$sqlArray [] = 'ALTER TABLE `servers` MODIFY COLUMN `organization` varchar(255) NOT NULL;' ;
break ;
case 'convertLogFieldsToText' :
$sqlArray [] = 'ALTER TABLE `logs` MODIFY COLUMN `title` text, MODIFY COLUMN `change` text;' ;
break ;
case 'addEventBlacklists' :
$sqlArray [] = 'CREATE TABLE IF NOT EXISTS `event_blacklists` ( `id` int(11) NOT NULL AUTO_INCREMENT, `event_uuid` varchar(40) COLLATE utf8_bin NOT NULL, `created` datetime NOT NULL, PRIMARY KEY (`id`), `event_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, `event_orgc` VARCHAR( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;' ;
break ;
case 'addOrgBlacklists' :
$sqlArray [] = 'CREATE TABLE IF NOT EXISTS `org_blacklists` ( `id` int(11) NOT NULL AUTO_INCREMENT, `org_uuid` varchar(40) COLLATE utf8_bin NOT NULL, `created` datetime NOT NULL, PRIMARY KEY (`id`), `org_name` varchar(255) COLLATE utf8_bin NOT NULL, `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;' ;
break ;
case 'addEventBlacklistsContext' :
$sqlArray [] = 'ALTER TABLE `event_blacklists` ADD `event_orgc` VARCHAR( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL , ADD `event_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, ADD `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL;' ;
break ;
case 'addSightings' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS sightings (
2019-02-10 13:08:12 +01:00
id int ( 11 ) NOT NULL AUTO_INCREMENT ,
attribute_id int ( 11 ) NOT NULL ,
event_id int ( 11 ) NOT NULL ,
org_id int ( 11 ) NOT NULL ,
date_sighting bigint ( 20 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX attribute_id ( attribute_id ),
INDEX event_id ( event_id ),
INDEX org_id ( org_id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 'makeAttributeUUIDsUnique' :
$this -> __dropIndex ( 'attributes' , 'uuid' );
$sqlArray [] = 'ALTER TABLE `attributes` ADD UNIQUE (uuid);' ;
break ;
case 'makeEventUUIDsUnique' :
$this -> __dropIndex ( 'events' , 'uuid' );
$sqlArray [] = 'ALTER TABLE `events` ADD UNIQUE (uuid);' ;
break ;
case 'cleanSessionTable' :
$sqlArray [] = 'DELETE FROM cake_sessions WHERE expires < ' . time () . ';' ;
$clean = false ;
break ;
case 'destroyAllSessions' :
$sqlArray [] = 'DELETE FROM cake_sessions;' ;
$clean = false ;
break ;
case 'addIPLogging' :
$sqlArray [] = 'ALTER TABLE `logs` ADD `ip` varchar(45) COLLATE utf8_bin DEFAULT NULL;' ;
break ;
case 'addCustomAuth' :
$sqlArray [] = " ALTER TABLE `users` ADD `external_auth_required` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `users` ADD `external_auth_key` text COLLATE utf8_bin;' ;
break ;
2018-10-22 10:36:13 +02:00
case 'x24betaupdates' :
2018-07-19 11:48:22 +02:00
$sqlArray = array ();
$sqlArray [] = " ALTER TABLE `shadow_attributes` ADD `proposal_to_delete` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `logs` MODIFY `change` text COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `taxonomies` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`namespace` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`description` text COLLATE utf8_bin NOT NULL ,
`version` int ( 11 ) NOT NULL ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `taxonomy_entries` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`taxonomy_predicate_id` int ( 11 ) NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`expanded` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` ),
KEY `taxonomy_predicate_id` ( `taxonomy_predicate_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `taxonomy_predicates` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`taxonomy_id` int ( 11 ) NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`expanded` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` ),
KEY `taxonomy_id` ( `taxonomy_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'ALTER TABLE `jobs` ADD `org` text COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `servers` ADD `name` varchar(255) NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `sharing_groups` ADD `sync_user_id` INT( 11 ) NOT NULL DEFAULT \'0\' AFTER `org_id`;' ;
$sqlArray [] = 'ALTER TABLE `users` ADD `disabled` BOOLEAN NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `users` ADD `expiration` datetime DEFAULT NULL;' ;
$sqlArray [] = 'UPDATE `roles` SET `perm_template` = 1 WHERE `perm_site_admin` = 1 OR `perm_admin` = 1;' ;
$sqlArray [] = 'UPDATE `roles` SET `perm_sharing_group` = 1 WHERE `perm_site_admin` = 1 OR `perm_sync` = 1;' ;
//create indexes
break ;
case 'indexTables' :
$fieldsToIndex = array (
'attributes' => array ( array ( 'value1' , 'INDEX' , '255' ), array ( 'value2' , 'INDEX' , '255' ), array ( 'event_id' , 'INDEX' ), array ( 'sharing_group_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' )),
'correlations' => array ( array ( 'org_id' , 'INDEX' ), array ( 'event_id' , 'INDEX' ), array ( 'attribute_id' , 'INDEX' ), array ( 'sharing_group_id' , 'INDEX' ), array ( '1_event_id' , 'INDEX' ), array ( '1_attribute_id' , 'INDEX' ), array ( 'a_sharing_group_id' , 'INDEX' ), array ( 'value' , 'FULLTEXT' )),
'events' => array ( array ( 'info' , 'FULLTEXT' ), array ( 'sharing_group_id' , 'INDEX' ), array ( 'org_id' , 'INDEX' ), array ( 'orgc_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' )),
'event_tags' => array ( array ( 'event_id' , 'INDEX' ), array ( 'tag_id' , 'INDEX' )),
'organisations' => array ( array ( 'uuid' , 'INDEX' ), array ( 'name' , 'FULLTEXT' )),
'posts' => array ( array ( 'post_id' , 'INDEX' ), array ( 'thread_id' , 'INDEX' )),
'shadow_attributes' => array ( array ( 'value1' , 'INDEX' , '255' ), array ( 'value2' , 'INDEX' , '255' ), array ( 'old_id' , 'INDEX' ), array ( 'event_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' ), array ( 'event_org_id' , 'INDEX' ), array ( 'event_uuid' , 'INDEX' )),
'sharing_groups' => array ( array ( 'org_id' , 'INDEX' ), array ( 'sync_user_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' ), array ( 'organisation_uuid' , 'INDEX' )),
'sharing_group_orgs' => array ( array ( 'sharing_group_id' , 'INDEX' ), array ( 'org_id' , 'INDEX' )),
'sharing_group_servers' => array ( array ( 'sharing_group_id' , 'INDEX' ), array ( 'server_id' , 'INDEX' )),
'servers' => array ( array ( 'org_id' , 'INDEX' ), array ( 'remote_org_id' , 'INDEX' )),
'tags' => array ( array ( 'name' , 'FULLTEXT' )),
'threads' => array ( array ( 'user_id' , 'INDEX' ), array ( 'event_id' , 'INDEX' ), array ( 'org_id' , 'INDEX' ), array ( 'sharing_group_id' , 'INDEX' )),
'users' => array ( array ( 'org_id' , 'INDEX' ), array ( 'server_id' , 'INDEX' ), array ( 'email' , 'INDEX' )),
);
$version = $this -> query ( 'select version();' );
$version = $version [ 0 ][ 0 ][ 'version()' ];
$version = explode ( '.' , $version );
$version [ 0 ] = intval ( $version [ 0 ]);
$version [ 1 ] = intval ( $version [ 1 ]);
$downgrade = true ;
if ( $version [ 0 ] > 5 || ( $version [ 0 ] == 5 && $version [ 1 ] > 5 )) {
$downgrade = false ;
}
// keep the fulltext for now, we can change it later to actually use it once we require MySQL 5.6 / or if we decide to move some tables to MyISAM
foreach ( $fieldsToIndex as $table => $fields ) {
$downgradeThis = false ;
$table_data = $this -> query ( " SHOW TABLE STATUS WHERE Name = ' " . $table . " ' " );
if ( $downgrade && $table_data [ 0 ][ 'TABLES' ][ 'Engine' ] !== 'MyISAM' ) {
$downgradeThis = true ;
}
foreach ( $fields as $field ) {
$extra = '' ;
$this -> __dropIndex ( $table , $field [ 0 ]);
if ( isset ( $field [ 2 ])) {
$extra = ' (' . $field [ 2 ] . ')' ;
}
$sqlArray [] = 'ALTER TABLE `' . $table . '` ADD ' . ( $downgradeThis ? 'INDEX' : $field [ 1 ]) . ' `' . $field [ 0 ] . '` (`' . $field [ 0 ] . '`' . $extra . ');' ;
}
}
break ;
case 'adminTable' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `admin_settings` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`setting` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = " INSERT INTO `admin_settings` (`setting`, `value`) VALUES ('db_version', '2.4.0'); " ;
break ;
case '2.4.18' :
$sqlArray [] = " ALTER TABLE `users` ADD `current_login` INT(11) DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ADD `last_login` INT(11) DEFAULT 0; " ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `event_delegations` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`requester_org_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`message` text ,
`distribution` tinyint ( 4 ) NOT NULL DEFAULT '-1' ,
`sharing_group_id` int ( 11 ),
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `event_id` ( `event_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.19' :
$sqlArray [] = " DELETE FROM `shadow_attributes` WHERE `event_uuid` = ''; " ;
break ;
case '2.4.20' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `shadow_attribute_correlations` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`value` text NOT NULL ,
`distribution` tinyint ( 4 ) NOT NULL ,
`a_distribution` tinyint ( 4 ) NOT NULL ,
`sharing_group_id` int ( 11 ),
`a_sharing_group_id` int ( 11 ),
`attribute_id` int ( 11 ) NOT NULL ,
`1_shadow_attribute_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`1_event_id` int ( 11 ) NOT NULL ,
`info` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `attribute_id` ( `attribute_id` ),
KEY `a_sharing_group_id` ( `a_sharing_group_id` ),
KEY `event_id` ( `event_id` ),
KEY `1_event_id` ( `event_id` ),
KEY `sharing_group_id` ( `sharing_group_id` ),
KEY `1_shadow_attribute_id` ( `1_shadow_attribute_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.25' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `feeds` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`provider` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`url` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`rules` text COLLATE utf8_bin NOT NULL ,
`enabled` BOOLEAN NOT NULL ,
`distribution` tinyint ( 4 ) NOT NULL ,
`sharing_group_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
`default` tinyint ( 1 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.32' :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_tag_editor` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'UPDATE `roles` SET `perm_tag_editor` = 1 WHERE `perm_tagger` = 1;' ;
break ;
case '2.4.33' :
$sqlArray [] = " ALTER TABLE `users` ADD `force_logout` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case '2.4.38' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `warninglists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT 'string' ,
`description` text COLLATE utf8_bin NOT NULL ,
`version` int ( 11 ) NOT NULL DEFAULT 1 ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`warninglist_entry_count` int ( 11 ) unsigned DEFAULT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `warninglist_entries` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`value` text CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`warninglist_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `warninglist_types` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`warninglist_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.39' :
$sqlArray [] = " ALTER TABLE `users` ADD `certif_public` longtext COLLATE utf8_bin AFTER `gpgkey`; " ;
$sqlArray [] = 'ALTER TABLE `logs` MODIFY COLUMN `title` text, MODIFY COLUMN `change` text;' ;
break ;
case '2.4.40' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `favourite_tags` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`tag_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` ),
INDEX `user_id` ( `user_id` ),
INDEX `tag_id` ( `tag_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.42' :
$sqlArray [] = " ALTER TABLE `attributes` ADD `deleted` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case '2.4.44' :
$sqlArray [] = " UPDATE `servers` SET `url` = TRIM(TRAILING '/' FROM `url`); " ;
break ;
case '2.4.45' :
$sqlArray [] = 'ALTER TABLE `users` CHANGE `newsread` `newsread` int(11) unsigned;' ;
$sqlArray [] = 'UPDATE `users` SET `newsread` = 0;' ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `news` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`message` text COLLATE utf8_bin NOT NULL ,
`title` text COLLATE utf8_bin NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`date_created` int ( 11 ) unsigned NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.49' :
// table: users
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `server_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `autoalert` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `invited_by` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `nids_sid` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `termsaccepted` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `role_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `change_pw` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `contactalert` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `disabled` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` MODIFY `authkey` varchar(40) COLLATE utf8_bin DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `users` MODIFY `gpgkey` longtext COLLATE utf8_bin; " ;
// table: events
$sqlArray [] = " ALTER TABLE `events` ALTER COLUMN `publish_timestamp` SET DEFAULT 0; " ;
// table: jobs
$sqlArray [] = " ALTER TABLE `jobs` ALTER COLUMN `org_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `jobs` MODIFY `process_id` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
// table: organisations
$sqlArray [] = " ALTER TABLE `organisations` ALTER COLUMN `created_by` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `organisations` MODIFY `uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL; " ; // https://github.com/MISP/MISP/pull/1260
// table: logs
$sqlArray [] = " ALTER TABLE `logs` MODIFY `title` text CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `logs` MODIFY `change` text CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `logs` MODIFY `description` text CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
// table: servers
$sqlArray [] = " ALTER TABLE `servers` DROP `lastfetchedid`; " ; // git commit hash d4c393897e8666fbbf04443a97d60c508700f5b4
$sqlArray [] = " ALTER TABLE `servers` MODIFY `cert_file` varchar(255) COLLATE utf8_bin DEFAULT NULL; " ;
// table: feeds
$sqlArray [] = " ALTER TABLE `feeds` ALTER COLUMN `sharing_group_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `feeds` ALTER COLUMN `tag_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `feeds` MODIFY `rules` text COLLATE utf8_bin DEFAULT NULL; " ;
// DB changes to support https://github.com/MISP/MISP/pull/1334
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_delegate` tinyint(1) NOT NULL DEFAULT 0 AFTER `perm_publish`; " ;
$sqlArray [] = " UPDATE `roles` SET `perm_delegate` = 1 WHERE `perm_publish` = 1; " ;
// DB changes to solve https://github.com/MISP/MISP/issues/1354
$sqlArray [] = " ALTER TABLE `taxonomy_entries` MODIFY `expanded` text COLLATE utf8_bin; " ;
$sqlArray [] = " ALTER TABLE `taxonomy_predicates` MODIFY `expanded` text COLLATE utf8_bin; " ;
// Sharing group propagate to instances freely setting
$sqlArray [] = " ALTER TABLE `sharing_groups` ADD `roaming` tinyint(1) NOT NULL DEFAULT 0; " ;
// table: shadow_attributes
$sqlArray [] = " ALTER TABLE `shadow_attributes` MODIFY `email` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL; " ;
// table: tasks
$sqlArray [] = " ALTER TABLE `tasks` CHANGE `job_id` `process_id` varchar(32) DEFAULT NULL; " ;
// Adding tag org restrictions
$sqlArray [] = " ALTER TABLE `tags` ADD `org_id` int(11) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `tags` ADD INDEX `org_id` (`org_id`);' ;
$this -> __dropIndex ( 'tags' , 'org_id' );
break ;
case '2.4.50' :
$sqlArray [] = 'ALTER TABLE `cake_sessions` ADD INDEX `expires` (`expires`);' ;
$sqlArray [] = " ALTER TABLE `users` ADD `certif_public` longtext COLLATE utf8_bin AFTER `gpgkey`; " ;
$sqlArray [] = " ALTER TABLE `servers` ADD `client_cert_file` varchar(255) COLLATE utf8_bin DEFAULT NULL; " ;
$this -> __dropIndex ( 'cake_sessions' , 'expires' );
break ;
case '2.4.51' :
$sqlArray [] = 'ALTER TABLE `servers` ADD `internal` tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE `roles` ADD `default_role` tinyint(1) NOT NULL DEFAULT 0;' ;
break ;
case '2.4.52' :
$sqlArray [] = " ALTER TABLE feeds ADD source_format varchar(255) COLLATE utf8_bin DEFAULT 'misp'; " ;
$sqlArray [] = 'ALTER TABLE feeds ADD fixed_event tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD delta_merge tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD event_id int(11) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD publish tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD override_ids tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = " ALTER TABLE feeds ADD settings text NOT NULL DEFAULT ''; " ;
break ;
case '2.4.56' :
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxies (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`description` text COLLATE utf8_bin NOT NULL ,
`version` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxies' , 'name' );
$this -> __addIndex ( 'galaxies' , 'uuid' );
$this -> __addIndex ( 'galaxies' , 'type' );
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxy_clusters (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`tag_name` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`description` text COLLATE utf8_bin NOT NULL ,
`galaxy_id` int ( 11 ) NOT NULL ,
`source` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`authors` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxy_clusters' , 'value' , 255 );
$this -> __addIndex ( 'galaxy_clusters' , 'tag_name' );
$this -> __addIndex ( 'galaxy_clusters' , 'uuid' );
$this -> __addIndex ( 'galaxy_clusters' , 'type' );
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxy_elements (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`galaxy_cluster_id` int ( 11 ) NOT NULL ,
`key` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`value` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxy_elements' , 'key' );
$this -> __addIndex ( 'galaxy_elements' , 'value' , 255 );
2016-12-04 20:25:47 +01:00
2018-07-19 11:48:22 +02:00
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxy_reference (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`galaxy_cluster_id` int ( 11 ) NOT NULL ,
`referenced_galaxy_cluster_id` int ( 11 ) NOT NULL ,
`referenced_galaxy_cluster_uuid` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`referenced_galaxy_cluster_type` text COLLATE utf8_bin NOT NULL ,
`referenced_galaxy_cluster_value` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxy_reference' , 'galaxy_cluster_id' );
$this -> __addIndex ( 'galaxy_reference' , 'referenced_galaxy_cluster_id' );
$this -> __addIndex ( 'galaxy_reference' , 'referenced_galaxy_cluster_value' , 255 );
$this -> __addIndex ( 'galaxy_reference' , 'referenced_galaxy_cluster_type' , 255 );
break ;
case '2.4.57' :
$sqlArray [] = 'ALTER TABLE tags ADD hide_tag tinyint(1) NOT NULL DEFAULT 0;' ;
// new indeces to match the changes in #1766
$this -> __dropIndex ( 'correlations' , '1_event_id' );
$this -> __addIndex ( 'correlations' , '1_event_id' );
$this -> __addIndex ( 'warninglist_entries' , 'warninglist_id' );
break ;
case '2.4.58' :
$sqlArray [] = " ALTER TABLE `events` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `attributes` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case '2.4.59' :
$sqlArray [] = " ALTER TABLE taxonomy_entries ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT ''; " ;
$sqlArray [] = " ALTER TABLE taxonomy_predicates ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT ''; " ;
break ;
case '2.4.60' :
2020-04-14 15:04:33 +02:00
if ( $dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver' ) {
2018-07-19 11:48:22 +02:00
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `attribute_tags` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`attribute_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; ' ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'ALTER TABLE `attribute_tags` ADD INDEX `attribute_id` (`attribute_id`);' ;
$sqlArray [] = 'ALTER TABLE `attribute_tags` ADD INDEX `event_id` (`event_id`);' ;
$sqlArray [] = 'ALTER TABLE `attribute_tags` ADD INDEX `tag_id` (`tag_id`);' ;
} elseif ( $dataSource == 'Database/Postgres' ) {
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS attribute_tags (
2019-02-10 13:08:12 +01:00
id bigserial NOT NULL ,
attribute_id bigint NOT NULL ,
event_id bigint NOT NULL ,
tag_id bigint NOT NULL ,
PRIMARY KEY ( id )
); ' ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'CREATE INDEX idx_attribute_tags_attribute_id ON attribute_tags (attribute_id);' ;
$sqlArray [] = 'CREATE INDEX idx_attribute_tags_event_id ON attribute_tags (event_id);' ;
$sqlArray [] = 'CREATE INDEX idx_attribute_tags_tag_id ON attribute_tags (tag_id);' ;
}
break ;
case '2.4.61' :
$sqlArray [] = 'ALTER TABLE feeds ADD input_source varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "network";' ;
$sqlArray [] = 'ALTER TABLE feeds ADD delete_local_file tinyint(1) DEFAULT 0;' ;
$indexArray [] = array ( 'feeds' , 'input_source' );
break ;
case '2.4.62' :
$sqlArray [] = 'ALTER TABLE logs CHANGE `org` `org` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE logs CHANGE `email` `email` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE logs CHANGE `change` `change` text COLLATE utf8_bin NOT NULL DEFAULT "";' ;
break ;
case '2.4.63' :
$sqlArray [] = 'ALTER TABLE events DROP COLUMN org;' ;
$sqlArray [] = 'ALTER TABLE events DROP COLUMN orgc;' ;
$sqlArray [] = 'ALTER TABLE event_blacklists CHANGE comment comment TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci;' ;
break ;
case '2.4.64' :
$indexArray [] = array ( 'feeds' , 'input_source' );
$indexArray [] = array ( 'attributes' , 'value1' , 255 );
$indexArray [] = array ( 'attributes' , 'value2' , 255 );
$indexArray [] = array ( 'attributes' , 'type' );
$indexArray [] = array ( 'galaxy_reference' , 'galaxy_cluster_id' );
$indexArray [] = array ( 'galaxy_reference' , 'referenced_galaxy_cluster_id' );
$indexArray [] = array ( 'galaxy_reference' , 'referenced_galaxy_cluster_value' , 255 );
$indexArray [] = array ( 'galaxy_reference' , 'referenced_galaxy_cluster_type' , 255 );
$indexArray [] = array ( 'correlations' , '1_event_id' );
$indexArray [] = array ( 'warninglist_entries' , 'warninglist_id' );
$indexArray [] = array ( 'galaxy_clusters' , 'value' , 255 );
$indexArray [] = array ( 'galaxy_clusters' , 'tag_name' );
$indexArray [] = array ( 'galaxy_clusters' , 'uuid' );
$indexArray [] = array ( 'galaxy_clusters' , 'type' );
$indexArray [] = array ( 'galaxies' , 'name' );
$indexArray [] = array ( 'galaxies' , 'uuid' );
$indexArray [] = array ( 'galaxies' , 'type' );
break ;
case '2.4.65' :
$sqlArray [] = 'ALTER TABLE feeds CHANGE `enabled` `enabled` tinyint(1) DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds CHANGE `default` `default` tinyint(1) DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds CHANGE `distribution` `distribution` tinyint(4) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds CHANGE `sharing_group_id` `sharing_group_id` int(11) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE attributes CHANGE `comment` `comment` text COLLATE utf8_bin;' ;
break ;
case '2.4.66' :
$sqlArray [] = 'ALTER TABLE shadow_attributes CHANGE old_id old_id int(11) DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE sightings ADD COLUMN uuid varchar(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE sightings ADD COLUMN source varchar(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE sightings ADD COLUMN type int(11) DEFAULT 0;' ;
$indexArray [] = array ( 'sightings' , 'uuid' );
$indexArray [] = array ( 'sightings' , 'source' );
$indexArray [] = array ( 'sightings' , 'type' );
$indexArray [] = array ( 'attributes' , 'category' );
$indexArray [] = array ( 'shadow_attributes' , 'category' );
$indexArray [] = array ( 'shadow_attributes' , 'type' );
break ;
case '2.4.67' :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_sighting` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'UPDATE `roles` SET `perm_sighting` = 1 WHERE `perm_add` = 1;' ;
break ;
case '2.4.68' :
$sqlArray [] = 'ALTER TABLE events CHANGE attribute_count attribute_count int(11) unsigned DEFAULT 0;' ;
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `event_blacklists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`created` datetime NOT NULL ,
`event_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`event_orgc` VARCHAR ( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; ' ;
2018-07-19 11:48:22 +02:00
$indexArray [] = array ( 'event_blacklists' , 'event_uuid' );
$indexArray [] = array ( 'event_blacklists' , 'event_orgc' );
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `org_blacklists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`created` datetime NOT NULL ,
`org_name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
PRIMARY KEY ( `id` ),
INDEX `org_uuid` ( `org_uuid` ),
INDEX `org_name` ( `org_name` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; ' ;
2018-07-19 11:48:22 +02:00
$indexArray [] = array ( 'org_blacklists' , 'org_uuid' );
$indexArray [] = array ( 'org_blacklists' , 'org_name' );
$sqlArray [] = " ALTER TABLE shadow_attributes CHANGE proposal_to_delete proposal_to_delete BOOLEAN DEFAULT 0 " ;
$sqlArray [] = " ALTER TABLE taxonomy_predicates CHANGE colour colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin; " ;
$sqlArray [] = " ALTER TABLE taxonomy_entries CHANGE colour colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin; " ;
break ;
case '2.4.69' :
$sqlArray [] = " ALTER TABLE taxonomy_entries CHANGE colour colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin; " ;
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_created bigint(20); " ;
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_modified bigint(20); " ;
break ;
case '2.4.71' :
$sqlArray [] = " UPDATE attributes SET comment = '' WHERE comment is NULL; " ;
$sqlArray [] = " ALTER TABLE attributes CHANGE comment comment text COLLATE utf8_bin NOT NULL; " ;
break ;
case '2.4.72' :
$sqlArray [] = 'ALTER TABLE feeds ADD lookup_visible tinyint(1) DEFAULT 0;' ;
break ;
case '2.4.73' :
$sqlArray [] = 'ALTER TABLE `servers` ADD `unpublish_event` tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE `servers` ADD `publish_without_email` tinyint(1) NOT NULL DEFAULT 0;' ;
break ;
case '2.4.75' :
$this -> __dropIndex ( 'attributes' , 'value1' );
$this -> __dropIndex ( 'attributes' , 'value2' );
$this -> __addIndex ( 'attributes' , 'value1' , 255 );
$this -> __addIndex ( 'attributes' , 'value2' , 255 );
break ;
case '2.4.77' :
$sqlArray [] = 'ALTER TABLE `users` CHANGE `password` `password` VARCHAR(255) COLLATE utf8_bin NOT NULL;' ;
break ;
case '2.4.78' :
$sqlArray [] = " ALTER TABLE galaxy_clusters ADD COLUMN version int(11) DEFAULT 0; " ;
$this -> __addIndex ( 'galaxy_clusters' , 'version' );
$this -> __addIndex ( 'galaxy_clusters' , 'galaxy_id' );
$this -> __addIndex ( 'galaxy_elements' , 'galaxy_cluster_id' );
break ;
case '2.4.80' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS objects (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`meta-category` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`description` text CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`template_uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`template_version` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`distribution` tinyint ( 4 ) NOT NULL DEFAULT 0 ,
`sharing_group_id` int ( 11 ),
`comment` text COLLATE utf8_bin NOT NULL ,
`deleted` TINYINT ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` ),
INDEX `template_uuid` ( `template_uuid` ),
INDEX `template_version` ( `template_version` ),
INDEX `meta-category` ( `meta-category` ),
INDEX `event_id` ( `event_id` ),
INDEX `uuid` ( `uuid` ),
INDEX `timestamp` ( `timestamp` ),
INDEX `distribution` ( `distribution` ),
INDEX `sharing_group_id` ( `sharing_group_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_references (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`object_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`object_uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`referenced_uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`referenced_id` int ( 11 ) NOT NULL ,
`referenced_type` int ( 11 ) NOT NULL DEFAULT 0 ,
`relationship_type` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`comment` text COLLATE utf8_bin NOT NULL ,
`deleted` TINYINT ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `object_uuid` ( `object_uuid` ),
INDEX `referenced_uuid` ( `referenced_uuid` ),
INDEX `timestamp` ( `timestamp` ),
INDEX `object_id` ( `object_id` ),
INDEX `referenced_id` ( `referenced_id` ),
INDEX `relationship_type` ( `relationship_type` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_relationships (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`version` int ( 11 ) NOT NULL ,
`name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`description` text COLLATE utf8_bin NOT NULL ,
`format` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-08-21 10:13:19 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_templates (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`meta-category` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`description` text COLLATE utf8_bin ,
`version` int ( 11 ) NOT NULL ,
`requirements` text COLLATE utf8_bin ,
`fixed` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`active` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` ),
INDEX `uuid` ( `uuid` ),
INDEX `name` ( `name` ),
INDEX `meta-category` ( `meta-category` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_template_elements (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`object_template_id` int ( 11 ) NOT NULL ,
`object_relation` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`type` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`ui-priority` int ( 11 ) NOT NULL ,
`categories` text COLLATE utf8_bin ,
`sane_default` text COLLATE utf8_bin ,
`values_list` text COLLATE utf8_bin ,
`description` text COLLATE utf8_bin ,
`disable_correlation` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`multiple` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `object_relation` ( `object_relation` ),
INDEX `type` ( `type` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'ALTER TABLE `logs` CHANGE `model` `model` VARCHAR(80) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `logs` CHANGE `action` `action` VARCHAR(80) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE attributes ADD object_id int(11) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE attributes ADD object_relation varchar(255) COLLATE utf8_bin;' ;
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_object_template` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'UPDATE `roles` SET `perm_object_template` = 1 WHERE `perm_site_admin` = 1;' ;
$indexArray [] = array ( 'attributes' , 'object_id' );
$indexArray [] = array ( 'attributes' , 'object_relation' );
break ;
case '2.4.81' :
$sqlArray [] = 'ALTER TABLE `galaxy_clusters` ADD `version` INT NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE `galaxies` ADD `icon` VARCHAR(255) COLLATE utf8_bin DEFAULT "";' ;
break ;
case '2.4.82' :
$sqlArray [] = " ALTER TABLE organisations ADD restricted_to_domain text COLLATE utf8_bin; " ;
break ;
case '2.4.83' :
$sqlArray [] = " ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` text COLLATE utf8_bin; " ;
break ;
case '2.4.84' :
$sqlArray [] = " ALTER TABLE `tags` ADD `user_id` int(11) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `tags` ADD INDEX `user_id` (`user_id`);' ;
break ;
case '2.4.85' :
$sqlArray [] = " ALTER TABLE `shadow_attributes` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` text COLLATE utf8_bin; " ;
// yes, this may look stupid as hell to index a boolean flag - but thanks to the stupidity of MySQL/MariaDB this will
// stop blocking other indexes to be used in queries where we also tests for the deleted flag.
$indexArray [] = array ( 'attributes' , 'deleted' );
break ;
case '2.4.86' :
break ;
case '2.4.87' :
$sqlArray [] = " ALTER TABLE `feeds` ADD `headers` TEXT COLLATE utf8_bin; " ;
break ;
case 1 :
$sqlArray [] = " ALTER TABLE `tags` ADD `user_id` int(11) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `tags` ADD INDEX `user_id` (`user_id`);' ;
break ;
case 2 :
// rerun missing db entries
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_created bigint(20); " ;
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_modified bigint(20); " ;
break ;
case 3 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `fuzzy_correlate_ssdeep` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`chunk` varchar ( 12 ) NOT NULL ,
`attribute_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'fuzzy_correlate_ssdeep' , 'chunk' );
$this -> __addIndex ( 'fuzzy_correlate_ssdeep' , 'attribute_id' );
break ;
case 4 :
$sqlArray [] = 'ALTER TABLE `roles` ADD `memory_limit` VARCHAR(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE `roles` ADD `max_execution_time` VARCHAR(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = " ALTER TABLE `roles` ADD `restricted_to_site_admin` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 5 :
$sqlArray [] = " ALTER TABLE `feeds` ADD `caching_enabled` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 6 :
$sqlArray [] = " ALTER TABLE `events` ADD `extends_uuid` varchar(40) COLLATE utf8_bin DEFAULT ''; " ;
$indexArray [] = array ( 'events' , 'extends_uuid' );
break ;
case 7 :
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `noticelists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) COLLATE utf8_unicode_ci NOT NULL ,
`expanded_name` text COLLATE utf8_unicode_ci NOT NULL ,
`ref` text COLLATE utf8_unicode_ci ,
`geographical_area` varchar ( 255 ) COLLATE utf8_unicode_ci ,
`version` int ( 11 ) NOT NULL DEFAULT 1 ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( `id` ),
INDEX `name` ( `name` ),
INDEX `geographical_area` ( `geographical_area` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; ' ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `noticelist_entries` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`noticelist_id` int ( 11 ) NOT NULL ,
`data` text COLLATE utf8_unicode_ci NOT NULL ,
PRIMARY KEY ( `id` ),
INDEX `noticelist_id` ( `noticelist_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; ' ;
2018-07-19 11:48:22 +02:00
break ;
case 9 :
$sqlArray [] = 'ALTER TABLE galaxies ADD namespace varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT "misp";' ;
$indexArray [] = array ( 'galaxies' , 'namespace' );
break ;
case 10 :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_publish_zmq` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 11 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_locks (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `event_id` ( `event_id` ),
INDEX `user_id` ( `user_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 12 :
$sqlArray [] = " ALTER TABLE `servers` ADD `skip_proxy` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 13 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_graph (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`network_name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`network_json` MEDIUMTEXT NOT NULL ,
`preview_img` MEDIUMTEXT ,
PRIMARY KEY ( id ),
INDEX `event_id` ( `event_id` ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 14 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `user_settings` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`setting` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
INDEX `setting` ( `setting` ),
INDEX `user_id` ( `user_id` ),
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 15 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_graph (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`network_name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`network_json` MEDIUMTEXT NOT NULL ,
`preview_img` MEDIUMTEXT ,
PRIMARY KEY ( id ),
INDEX `event_id` ( `event_id` ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
2018-09-10 07:52:27 +02:00
case 18 :
2018-07-22 23:28:07 +02:00
$sqlArray [] = 'ALTER TABLE `taxonomy_predicates` ADD COLUMN description text CHARACTER SET UTF8 collate utf8_bin;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_entries` ADD COLUMN description text CHARACTER SET UTF8 collate utf8_bin;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_predicates` ADD COLUMN exclusive tinyint(1) DEFAULT 0;' ;
break ;
2018-09-10 07:52:27 +02:00
case 19 :
2018-07-26 17:13:38 +02:00
$sqlArray [] = 'ALTER TABLE `taxonomies` ADD COLUMN exclusive tinyint(1) DEFAULT 0;' ;
break ;
2018-09-10 22:02:07 +02:00
case 20 :
$sqlArray [] = " ALTER TABLE `servers` ADD `skip_proxy` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 21 :
$sqlArray [] = 'ALTER TABLE `tags` ADD COLUMN numerical_value int(11) NULL;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_predicates` ADD COLUMN numerical_value int(11) NULL;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_entries` ADD COLUMN numerical_value int(11) NULL;' ;
break ;
2018-11-23 14:11:33 +01:00
case 22 :
$sqlArray [] = 'ALTER TABLE `object_references` MODIFY `deleted` tinyint(1) NOT NULL default 0;' ;
break ;
case 24 :
$this -> GalaxyCluster = ClassRegistry :: init ( 'GalaxyCluster' );
if ( empty ( $this -> GalaxyCluster -> schema ( 'collection_uuid' ))) {
$sqlArray [] = 'ALTER TABLE `galaxy_clusters` CHANGE `uuid` `collection_uuid` varchar(255) COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `galaxy_clusters` ADD COLUMN `uuid` varchar(255) COLLATE utf8_bin NOT NULL default \'\';' ;
}
break ;
case 25 :
$this -> __dropIndex ( 'galaxy_clusters' , 'uuid' );
$this -> __addIndex ( 'galaxy_clusters' , 'uuid' );
$this -> __addIndex ( 'galaxy_clusters' , 'collection_uuid' );
break ;
2018-12-15 10:02:38 +01:00
case 26 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS tag_collections (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`name` varchar ( 255 ) COLLATE utf8_unicode_ci NOT NULL ,
`description` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`all_orgs` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `uuid` ( `uuid` ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-12-27 05:24:30 +01:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS tag_collection_tags (
2018-12-15 10:02:38 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`tag_collection_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `uuid` ( `tag_collection_id` ),
INDEX `user_id` ( `tag_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
break ;
2019-01-01 16:37:15 +01:00
case 27 :
$sqlArray [] = 'ALTER TABLE `tags` CHANGE `org_id` `org_id` int(11) NOT NULL DEFAULT 0;' ;
break ;
2019-01-18 09:06:23 +01:00
case 28 :
$sqlArray [] = " ALTER TABLE `servers` ADD `caching_enabled` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-02-14 10:24:42 +01:00
case 29 :
$sqlArray [] = " ALTER TABLE `galaxies` ADD `kill_chain_order` text NOT NULL; " ;
break ;
2019-02-22 08:04:23 +01:00
case 30 :
$sqlArray [] = " ALTER TABLE `galaxies` MODIFY COLUMN `kill_chain_order` text " ;
$sqlArray [] = " ALTER TABLE `feeds` ADD `force_to_ids` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-03-19 10:55:27 +01:00
case 31 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `rest_client_histories` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`headers` text ,
`body` text ,
`url` text ,
`http_method` varchar ( 255 ),
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`use_full_path` tinyint ( 1 ) DEFAULT 0 ,
`show_result` tinyint ( 1 ) DEFAULT 0 ,
`skip_ssl` tinyint ( 1 ) DEFAULT 0 ,
`outcome` int ( 11 ) NOT NULL ,
`bookmark` tinyint ( 1 ) NOT NULL DEFAUlT 0 ,
`bookmark_name` varchar ( 255 ) NULL DEFAULT '' ,
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `user_id` ( `user_id` ),
KEY `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
break ;
2019-03-19 15:59:31 +01:00
case 32 :
$sqlArray [] = " ALTER TABLE `taxonomies` ADD `required` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-03-19 17:20:46 +01:00
case 33 :
2019-03-05 12:24:56 +01:00
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_publish_kafka` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-06-04 11:30:01 +02:00
case 35 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `notification_logs` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `type` ( `type` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2019-09-12 10:22:13 +02:00
break ;
2019-07-08 11:39:41 +02:00
case 36 :
$sqlArray [] = " ALTER TABLE `event_tags` ADD `local` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `attribute_tags` ADD `local` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-09-13 11:49:12 +02:00
case 37 :
2019-03-08 09:22:37 +01:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS decaying_models (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
2019-04-03 10:05:45 +02:00
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
2019-09-12 14:44:09 +02:00
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
2019-03-08 09:22:37 +01:00
`parameters` text ,
2019-04-03 11:51:29 +02:00
`attribute_types` text ,
2019-03-08 09:22:37 +01:00
`description` text ,
2019-04-03 10:05:45 +02:00
`org_id` int ( 11 ),
2019-07-24 14:36:13 +02:00
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
2019-08-16 10:33:58 +02:00
`all_orgs` tinyint ( 1 ) NOT NULL DEFAULT 1 ,
2019-04-03 11:51:29 +02:00
`ref` text COLLATE utf8_unicode_ci ,
`formula` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
2019-08-29 12:22:35 +02:00
`version` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
2019-08-30 09:18:00 +02:00
`default` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
2019-09-12 14:28:49 +02:00
PRIMARY KEY ( id ),
INDEX `uuid` ( `uuid` ),
INDEX `name` ( `name` ),
INDEX `org_id` ( `org_id` ),
INDEX `enabled` ( `enabled` ),
INDEX `all_orgs` ( `all_orgs` ),
INDEX `version` ( `version` )
2019-04-03 10:05:45 +02:00
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS decaying_model_mappings (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`attribute_type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`model_id` int ( 11 ) NOT NULL ,
2019-09-12 14:28:49 +02:00
PRIMARY KEY ( id ),
INDEX `model_id` ( `model_id` )
2019-03-08 09:22:37 +01:00
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2019-08-20 16:37:06 +02:00
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_decaying` tinyint(1) NOT NULL DEFAULT 0; " ;
2019-08-27 09:19:44 +02:00
$sqlArray [] = " UPDATE `roles` SET `perm_decaying`=1 WHERE `perm_sighting`=1; " ;
2019-07-08 11:39:41 +02:00
break ;
2019-09-13 11:51:05 +02:00
case 38 :
2019-09-13 11:49:12 +02:00
$sqlArray [] = " ALTER TABLE servers ADD priority int(11) NOT NULL DEFAULT 0; " ;
$indexArray [] = array ( 'servers' , 'priority' );
break ;
2019-09-25 11:50:54 +02:00
case 39 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS user_settings (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
2020-04-29 16:13:54 +02:00
`setting` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
2019-09-25 11:50:54 +02:00
`value` text ,
`user_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `key` ( `key` ),
INDEX `user_id` ( `user_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2019-09-29 22:20:46 +02:00
case 40 :
$sqlArray [] = " ALTER TABLE `user_settings` ADD `timestamp` int(11) NOT NULL; " ;
$indexArray [] = array ( 'user_settings' , 'timestamp' );
break ;
2019-10-08 11:43:56 +02:00
case 41 :
$sqlArray [] = " ALTER TABLE `roles` ADD `enforce_rate_limit` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `roles` ADD `rate_limit_count` int(11) NOT NULL DEFAULT 0; " ;
break ;
2019-11-06 21:20:04 +01:00
case 42 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS sightingdbs (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) NOT NULL ,
`description` text ,
`owner` varchar ( 255 ) DEFAULT '' ,
`host` varchar ( 255 ) DEFAULT 'http://localhost' ,
`port` int ( 11 ) DEFAULT 9999 ,
`timestamp` int ( 11 ) NOT NULL ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`skip_proxy` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`ssl_skip_verification` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` ),
INDEX `owner` ( `owner` ),
INDEX `host` ( `host` ),
INDEX `port` ( `port` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS sightingdb_orgs (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`sightingdb_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `sightingdb_id` ( `sightingdb_id` ),
INDEX `org_id` ( `org_id` )
) ENGINE = InnoDB ; " ;
break ;
2019-11-08 08:24:04 +01:00
case 43 :
$sqlArray [] = " ALTER TABLE sightingdbs ADD namespace varchar(255) DEFAULT ''; " ;
2019-11-18 11:35:10 +01:00
break ;
case 44 :
$sqlArray [] = " ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1); " ;
2019-11-22 21:53:51 +01:00
break ;
case 45 :
$sqlArray [] = " ALTER TABLE `events` ADD `sighting_timestamp` int(11) NOT NULL DEFAULT 0 AFTER `publish_timestamp`; " ;
$sqlArray [] = " ALTER TABLE `servers` ADD `push_sightings` tinyint(1) NOT NULL DEFAULT 0 AFTER `pull`; " ;
2019-11-08 08:24:04 +01:00
break ;
2020-02-07 12:56:27 +01:00
case 47 :
$this -> __addIndex ( 'tags' , 'numerical_value' );
$this -> __addIndex ( 'taxonomy_predicates' , 'numerical_value' );
$this -> __addIndex ( 'taxonomy_entries' , 'numerical_value' );
break ;
2020-03-08 23:36:27 +01:00
case 49 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS dashboards (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`name` varchar ( 191 ) NOT NULL ,
`description` text ,
`default` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`selectable` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`user_id` int ( 11 ) NOT NULL DEFAULT 0 ,
`restrict_to_org_id` int ( 11 ) NOT NULL DEFAULT 0 ,
`restrict_to_role_id` int ( 11 ) NOT NULL DEFAULT 0 ,
`restrict_to_permission_flag` varchar ( 191 ) NOT NULL DEFAULT '' ,
`value` text ,
`timestamp` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` ),
INDEX `uuid` ( `uuid` ),
INDEX `user_id` ( `user_id` ),
INDEX `restrict_to_org_id` ( `restrict_to_org_id` ),
INDEX `restrict_to_permission_flag` ( `restrict_to_permission_flag` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2020-04-07 13:21:01 +02:00
case 50 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS inbox (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`title` varchar ( 191 ) NOT NULL ,
`type` varchar ( 191 ) NOT NULL ,
`ip` varchar ( 191 ) NOT NULL ,
`user_agent` text ,
`user_agent_sha256` varchar ( 64 ) NOT NULL ,
`comment` text ,
`deleted` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`timestamp` int ( 11 ) NOT NULL ,
`store_as_file` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`data` longtext ,
PRIMARY KEY ( id ),
INDEX `title` ( `title` ),
INDEX `type` ( `type` ),
INDEX `uuid` ( `uuid` ),
INDEX `user_agent_sha256` ( `user_agent_sha256` ),
INDEX `ip` ( `ip` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2020-04-17 14:17:54 +02:00
case 51 :
$sqlArray [] = " ALTER TABLE `feeds` ADD `orgc_id` int(11) NOT NULL DEFAULT 0 " ;
$this -> __addIndex ( 'feeds' , 'orgc_id' );
break ;
2020-04-29 16:13:54 +02:00
case 52 :
if ( ! empty ( $this -> query ( " SHOW COLUMNS FROM `admin_settings` LIKE 'key'; " ))) {
$sqlArray [] = " ALTER TABLE admin_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL; " ;
$this -> __addIndex ( 'admin_settings' , 'setting' );
}
break ;
2020-04-30 06:46:46 +02:00
case 53 :
if ( ! empty ( $this -> query ( " SHOW COLUMNS FROM `user_settings` LIKE 'key'; " ))) {
$sqlArray [] = " ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL; " ;
$this -> __addIndex ( 'user_settings' , 'setting' );
}
break ;
2020-05-14 15:23:28 +02:00
case 54 :
$sqlArray [] = " ALTER TABLE `sightingdbs` MODIFY `timestamp` int(11) NOT NULL DEFAULT 0; " ;
break ;
2020-06-07 18:54:42 +02:00
case 55 :
2020-06-07 19:16:33 +02:00
// index is not used in any SQL query
$this -> __dropIndex ( 'correlations' , 'value' );
// these index can be theoretically used, but probably just in very rare occasion
$this -> __dropIndex ( 'correlations' , 'org_id' );
$this -> __dropIndex ( 'correlations' , 'sharing_group_id' );
$this -> __dropIndex ( 'correlations' , 'a_sharing_group_id' );
2020-06-07 18:54:42 +02:00
break ;
2018-07-19 11:48:22 +02:00
case 'fixNonEmptySharingGroupID' :
$sqlArray [] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;' ;
$sqlArray [] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;' ;
break ;
case 'cleanupAfterUpgrade' :
$sqlArray [] = 'ALTER TABLE `events` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `events` DROP `orgc`;' ;
$sqlArray [] = 'ALTER TABLE `correlations` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `jobs` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `servers` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `servers` DROP `organization`;' ;
$sqlArray [] = 'ALTER TABLE `shadow_attributes` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `shadow_attributes` DROP `event_org`;' ;
$sqlArray [] = 'ALTER TABLE `threads` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `users` DROP `org`;' ;
break ;
2019-06-13 09:16:34 +02:00
case 'seenOnAttributeAndObject' :
$sqlArray [] =
" ALTER TABLE `attributes`
DROP INDEX uuid ,
DROP INDEX event_id ,
DROP INDEX sharing_group_id ,
DROP INDEX type ,
DROP INDEX category ,
DROP INDEX value1 ,
DROP INDEX value2 ,
DROP INDEX object_id ,
2019-06-13 09:38:39 +02:00
DROP INDEX object_relation ;
" ;
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX deleted " ; // deleted index may not be present
2019-07-02 16:26:05 +02:00
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX comment " ; // for replayability
2019-06-13 09:38:39 +02:00
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX first_seen " ; // for replayability
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX last_seen " ; // for replayability
2019-06-13 09:16:34 +02:00
$sqlArray [] =
" ALTER TABLE `attributes`
ADD COLUMN `first_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
ADD COLUMN `last_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
MODIFY comment TEXT COLLATE utf8_unicode_ci
; " ;
2019-07-01 14:35:56 +02:00
$indexArray [] = array ( 'attributes' , 'uuid' );
$indexArray [] = array ( 'attributes' , 'event_id' );
$indexArray [] = array ( 'attributes' , 'sharing_group_id' );
$indexArray [] = array ( 'attributes' , 'type' );
$indexArray [] = array ( 'attributes' , 'category' );
2019-12-16 14:59:53 +01:00
$indexArray [] = array ( 'attributes' , 'value1' , 255 );
$indexArray [] = array ( 'attributes' , 'value2' , 255 );
2019-07-01 14:35:56 +02:00
$indexArray [] = array ( 'attributes' , 'object_id' );
$indexArray [] = array ( 'attributes' , 'object_relation' );
$indexArray [] = array ( 'attributes' , 'deleted' );
$indexArray [] = array ( 'attributes' , 'first_seen' );
$indexArray [] = array ( 'attributes' , 'last_seen' );
2019-06-13 09:16:34 +02:00
$sqlArray [] = "
ALTER TABLE `objects`
ADD `first_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
2019-07-02 16:26:05 +02:00
ADD `last_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
MODIFY comment TEXT COLLATE utf8_unicode_ci
2019-06-13 09:16:34 +02:00
; " ;
$indexArray [] = array ( 'objects' , 'first_seen' );
$indexArray [] = array ( 'objects' , 'last_seen' );
2019-06-24 10:28:55 +02:00
$sqlArray [] = "
ALTER TABLE `shadow_attributes`
ADD `first_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
2019-07-02 16:26:05 +02:00
ADD `last_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
MODIFY comment TEXT COLLATE utf8_unicode_ci
2019-06-24 10:28:55 +02:00
; " ;
$indexArray [] = array ( 'shadow_attributes' , 'first_seen' );
$indexArray [] = array ( 'shadow_attributes' , 'last_seen' );
2019-06-13 09:16:34 +02:00
break ;
2018-07-19 11:48:22 +02:00
default :
return false ;
break ;
}
2019-04-26 09:45:03 +02:00
$now = new DateTime ();
// switch MISP instance live to false
if ( $liveOff ) {
$this -> Server = Classregistry :: init ( 'Server' );
$liveSetting = 'MISP.live' ;
$this -> Server -> serverSettingsSaveValue ( $liveSetting , false );
}
2019-04-29 11:09:04 +02:00
$sql_update_count = count ( $sqlArray );
$index_update_count = count ( $indexArray );
$total_update_count = $sql_update_count + $index_update_count ;
2019-10-03 13:50:55 +02:00
$this -> __setUpdateProgress ( 0 , $total_update_count , $command );
2019-04-29 11:09:04 +02:00
$str_index_array = array ();
2019-04-26 09:45:03 +02:00
foreach ( $indexArray as $toIndex ) {
2019-12-16 14:59:53 +01:00
$str_index_array [] = __ ( 'Indexing ' ) . sprintf ( '%s -> %s' , $toIndex [ 0 ], $toIndex [ 1 ]);
2019-04-26 09:45:03 +02:00
}
2019-04-29 11:09:04 +02:00
$this -> __setUpdateCmdMessages ( array_merge ( $sqlArray , $str_index_array ));
2019-10-29 09:57:25 +01:00
$flagStop = false ;
$errorCount = 0 ;
2019-04-29 11:26:55 +02:00
// execute test before update. Exit if it fails
if ( isset ( $this -> advanced_updates_description [ $command ][ 'preUpdate' ])) {
$function_name = $this -> advanced_updates_description [ $command ][ 'preUpdate' ];
2018-07-19 11:48:22 +02:00
try {
2019-04-29 11:26:55 +02:00
$this -> { $function_name }();
} catch ( Exception $e ) {
$this -> __setPreUpdateTestState ( false );
$this -> __setUpdateProgress ( 0 , false );
2019-10-02 15:07:37 +02:00
$this -> __setUpdateResMessages ( 0 , sprintf ( __ ( 'Issues executing the pre-update test `%s`. The returned error is: %s' ), $function_name , $e -> getMessage ()) . PHP_EOL );
2019-04-29 11:26:55 +02:00
$this -> __setUpdateError ( 0 );
2019-10-29 09:57:25 +01:00
$errorCount ++ ;
2019-04-29 11:26:55 +02:00
$exitOnError = true ;
2019-10-29 09:57:25 +01:00
$flagStop = true ;
2019-04-29 11:26:55 +02:00
}
}
2019-10-29 09:57:25 +01:00
if ( ! $flagStop ) {
2019-04-29 11:26:55 +02:00
$this -> __setPreUpdateTestState ( true );
foreach ( $sqlArray as $i => $sql ) {
try {
$this -> __setUpdateProgress ( $i , false );
$this -> query ( $sql );
$this -> Log -> create ();
$this -> Log -> save ( array (
2018-07-19 11:48:22 +02:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
2019-04-29 11:09:04 +02:00
'title' => __ ( 'Successfuly executed the SQL query for ' ) . $command ,
2019-10-01 15:46:45 +02:00
'change' => sprintf ( __ ( 'The executed SQL query was: %s' ), $sql )
2019-04-29 11:26:55 +02:00
));
2019-10-01 15:46:45 +02:00
$this -> __setUpdateResMessages ( $i , sprintf ( __ ( 'Successfuly executed the SQL query for %s' ), $command ));
2019-04-29 11:26:55 +02:00
} catch ( Exception $e ) {
2019-10-29 09:57:25 +01:00
$errorMessage = $e -> getMessage ();
2019-04-29 11:26:55 +02:00
$this -> Log -> create ();
2019-10-29 09:57:25 +01:00
$logMessage = array (
2018-07-19 11:48:22 +02:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
2019-10-01 15:46:45 +02:00
'title' => sprintf ( __ ( 'Issues executing the SQL query for %s' ), $command ),
2019-10-29 09:57:25 +01:00
'change' => __ ( 'The executed SQL query was: ' ) . $sql . PHP_EOL . __ ( ' The returned error is: ' ) . $errorMessage
2019-09-30 10:30:41 +02:00
);
2019-10-29 09:57:25 +01:00
$this -> __setUpdateResMessages ( $i , sprintf ( __ ( 'Issues executing the SQL query for `%s`. The returned error is: ' . PHP_EOL . '%s' ), $command , $errorMessage ));
if ( ! $this -> isAcceptedDatabaseError ( $errorMessage , $dataSource )) {
2019-08-14 12:13:45 +02:00
$this -> __setUpdateError ( $i );
2019-10-29 09:57:25 +01:00
$errorCount ++ ;
2019-08-14 12:13:45 +02:00
if ( $exitOnError ) {
2019-10-29 09:57:25 +01:00
$flagStop = true ;
2019-08-14 12:13:45 +02:00
break ;
}
2019-09-30 10:30:41 +02:00
} else {
2019-10-29 09:57:25 +01:00
$logMessage [ 'change' ] = $logMessage [ 'change' ] . PHP_EOL . __ ( 'However, as this error is whitelisted, the update went through.' );
2019-04-29 11:26:55 +02:00
}
2019-10-29 09:57:25 +01:00
$this -> Log -> save ( $logMessage );
2018-07-19 11:48:22 +02:00
}
}
}
2019-10-29 09:57:25 +01:00
if ( ! $flagStop ) {
2019-09-30 10:30:41 +02:00
if ( ! empty ( $indexArray )) {
if ( $clean ) {
$this -> cleanCacheFiles ();
}
foreach ( $indexArray as $i => $iA ) {
$this -> __setUpdateProgress ( count ( $sqlArray ) + $i , false );
if ( isset ( $iA [ 2 ])) {
2019-12-16 14:59:53 +01:00
$indexSuccess = $this -> __addIndex ( $iA [ 0 ], $iA [ 1 ], $iA [ 2 ]);
2019-09-30 10:30:41 +02:00
} else {
2019-12-16 14:59:53 +01:00
$indexSuccess = $this -> __addIndex ( $iA [ 0 ], $iA [ 1 ]);
}
if ( $indexSuccess [ 'success' ]) {
$this -> __setUpdateResMessages ( count ( $sqlArray ) + $i , __ ( 'Successfuly indexed ' ) . sprintf ( '%s -> %s' , $iA [ 0 ], $iA [ 1 ]));
} else {
2020-01-20 10:23:51 +01:00
$this -> __setUpdateResMessages ( count ( $sqlArray ) + $i , sprintf ( '%s %s %s %s' ,
2019-12-16 14:59:53 +01:00
__ ( 'Failed to add index' ),
sprintf ( '%s -> %s' , $iA [ 0 ], $iA [ 1 ]),
__ ( 'The returned error is:' ) . PHP_EOL ,
$indexSuccess [ 'errorMessage' ]
));
$this -> __setUpdateError ( count ( $sqlArray ) + $i );
2019-09-30 10:30:41 +02:00
}
}
}
$this -> __setUpdateProgress ( count ( $sqlArray ) + count ( $indexArray ), false );
2019-04-26 09:45:03 +02:00
}
2018-07-19 11:48:22 +02:00
if ( $clean ) {
$this -> cleanCacheFiles ();
}
2019-04-26 09:45:03 +02:00
if ( $liveOff ) {
$liveSetting = 'MISP.live' ;
$this -> Server -> serverSettingsSaveValue ( $liveSetting , true );
}
2019-10-29 09:57:25 +01:00
if ( ! $flagStop && $errorCount == 0 ) {
2019-04-26 09:45:03 +02:00
$this -> __postUpdate ( $command );
}
2019-10-29 09:57:25 +01:00
if ( $flagStop && $errorCount > 0 ) {
2019-08-14 12:13:45 +02:00
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
2019-10-01 15:46:45 +02:00
'title' => sprintf ( __ ( 'Issues executing the SQL query for %s' ), $command ),
2019-08-14 12:13:45 +02:00
'change' => __ ( 'Database updates stopped as some errors occured and the stop flag is enabled.' )
));
return false ;
}
2018-07-19 11:48:22 +02:00
return true ;
}
2019-04-26 09:45:03 +02:00
// check whether the adminSetting should be updated after the update
private function __postUpdate ( $command ) {
if ( isset ( $this -> advanced_updates_description [ $command ][ 'record' ])) {
if ( $this -> advanced_updates_description [ $command ][ 'record' ]) {
$this -> AdminSetting -> changeSetting ( $command , 1 );
}
}
}
2018-07-19 11:48:22 +02:00
private function __dropIndex ( $table , $field )
{
$dataSourceConfig = ConnectionManager :: getDataSource ( 'default' ) -> config ;
$dataSource = $dataSourceConfig [ 'datasource' ];
$this -> Log = ClassRegistry :: init ( 'Log' );
$indexCheckResult = array ();
2020-04-14 15:04:33 +02:00
if ( $dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver' ) {
2018-07-19 11:48:22 +02:00
$indexCheck = " SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name=' " . $table . " ' AND index_name LIKE ' " . $field . " %'; " ;
$indexCheckResult = $this -> query ( $indexCheck );
} elseif ( $dataSource == 'Database/Postgres' ) {
$pgIndexName = 'idx_' . $table . '_' . $field ;
$indexCheckResult [] = array ( 'STATISTICS' => array ( 'INDEX_NAME' => $pgIndexName ));
}
foreach ( $indexCheckResult as $icr ) {
2020-04-14 15:04:33 +02:00
if ( $dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver' ) {
2018-07-19 11:48:22 +02:00
$dropIndex = 'ALTER TABLE ' . $table . ' DROP INDEX ' . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ';' ;
} elseif ( $dataSource == 'Database/Postgres' ) {
$dropIndex = 'DROP INDEX IF EXISTS ' . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ';' ;
}
$result = true ;
try {
$this -> query ( $dropIndex );
} catch ( Exception $e ) {
$result = false ;
}
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => ( $result ? 'Removed index ' : 'Failed to remove index ' ) . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ' from ' . $table ,
'change' => ( $result ? 'Removed index ' : 'Failed to remove index ' ) . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ' from ' . $table ,
));
}
}
private function __addIndex ( $table , $field , $length = false )
{
$dataSourceConfig = ConnectionManager :: getDataSource ( 'default' ) -> config ;
$dataSource = $dataSourceConfig [ 'datasource' ];
$this -> Log = ClassRegistry :: init ( 'Log' );
if ( $dataSource == 'Database/Postgres' ) {
$addIndex = " CREATE INDEX idx_ " . $table . " _ " . $field . " ON " . $table . " ( " . $field . " ); " ;
} else {
if ( ! $length ) {
$addIndex = " ALTER TABLE ` " . $table . " ` ADD INDEX ` " . $field . " ` (` " . $field . " `); " ;
} else {
$addIndex = " ALTER TABLE ` " . $table . " ` ADD INDEX ` " . $field . " ` (` " . $field . " `( " . $length . " )); " ;
}
}
$result = true ;
$duplicate = false ;
2019-12-16 14:59:53 +01:00
$errorMessage = '' ;
2018-07-19 11:48:22 +02:00
try {
$this -> query ( $addIndex );
} catch ( Exception $e ) {
$duplicate = ( strpos ( $e -> getMessage (), '1061' ) !== false );
2019-12-16 14:59:53 +01:00
$errorMessage = $e -> getMessage ();
2018-07-19 11:48:22 +02:00
$result = false ;
}
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
2019-12-16 14:59:53 +01:00
'title' => ( $result ? 'Added index ' : 'Failed to add index ' ) . $field . ' to ' . $table . ( $duplicate ? ' (index already set)' : $errorMessage ),
'change' => ( $result ? 'Added index ' : 'Failed to add index ' ) . $field . ' to ' . $table . ( $duplicate ? ' (index already set)' : $errorMessage ),
2018-07-19 11:48:22 +02:00
));
2019-12-16 14:59:53 +01:00
$additionResult = array ( 'success' => $result || $duplicate );
if ( ! $result ) {
$additionResult [ 'errorMessage' ] = $errorMessage ;
}
return $additionResult ;
2018-07-19 11:48:22 +02:00
}
public function cleanCacheFiles ()
{
Cache :: clear ();
2019-09-16 19:41:27 +02:00
Cache :: clear ( false , '_cake_core_' );
Cache :: clear ( false , '_cake_model_' );
2018-07-19 11:48:22 +02:00
clearCache ();
2019-09-16 19:41:27 +02:00
$files = glob ( CACHE . 'models' . DS . 'myapp*' );
2018-07-19 11:48:22 +02:00
$files = array_merge ( $files , glob ( CACHE . 'persistent' . DS . 'myapp*' ));
2019-09-16 19:41:27 +02:00
foreach ( $files as $file ) {
if ( is_file ( $file )) {
unlink ( $file );
2018-07-19 11:48:22 +02:00
}
}
}
public function checkMISPVersion ()
{
$file = new File ( ROOT . DS . 'VERSION.json' , true );
$version_array = json_decode ( $file -> read (), true );
$file -> close ();
return $version_array ;
}
2018-09-25 21:29:34 +02:00
public function getPythonVersion ()
{
if ( ! empty ( Configure :: read ( 'MISP.python_bin' ))) {
return Configure :: read ( 'MISP.python_bin' );
} else {
return 'python3' ;
}
}
2018-08-20 10:50:09 +02:00
public function validateAuthkey ( $value )
{
2018-08-03 18:40:46 +02:00
if ( empty ( $value [ 'authkey' ])) {
return 'Empty authkey found. Make sure you set the 40 character long authkey.' ;
}
if ( ! preg_match ( '/[a-z0-9]{40}/i' , $value [ 'authkey' ])) {
return 'The authkey has to be exactly 40 characters long and consist of alphanumeric characters.' ;
}
return true ;
}
2018-07-19 11:48:22 +02:00
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty ( $value )
{
$field = array_keys ( $value );
$field = $field [ 0 ];
$value [ $field ] = trim ( $value [ $field ]);
if ( ! empty ( $value [ $field ])) {
return true ;
}
return ucfirst ( $field ) . ' cannot be empty.' ;
}
2019-09-29 20:20:39 +02:00
public function valueIsJson ( $value )
{
$field = array_keys ( $value );
$field = $field [ 0 ];
$json_decoded = json_decode ( $value [ $field ]);
if ( $json_decoded === null ) {
return __ ( 'Invalid JSON.' );
}
return true ;
}
2018-07-19 11:48:22 +02:00
public function valueIsID ( $value )
{
$field = array_keys ( $value );
$field = $field [ 0 ];
if ( ! is_numeric ( $value [ $field ]) || $value [ $field ] < 0 ) {
return 'Invalid ' . ucfirst ( $field ) . ' ID' ;
}
return true ;
}
public function stringNotEmpty ( $value )
{
$field = array_keys ( $value );
$field = $field [ 0 ];
$value [ $field ] = trim ( $value [ $field ]);
if ( ! isset ( $value [ $field ]) || ( $value [ $field ] == false && $value [ $field ] !== " 0 " )) {
return ucfirst ( $field ) . ' cannot be empty.' ;
}
return true ;
}
2019-06-13 09:16:34 +02:00
// Try to create a table with a BIGINT(20)
public function seenOnAttributeAndObjectPreUpdate () {
$sqlArray [] = " CREATE TABLE IF NOT EXISTS testtable (
`testfield` BIGINT ( 6 ) NULL DEFAULT NULL
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
try {
foreach ( $sqlArray as $i => $sql ) {
$this -> query ( $sql );
}
} catch ( Exception $e ) {
throw new Exception ( 'Pre update test failed: ' . PHP_EOL . $sql . PHP_EOL . ' The returned error is: ' . $e -> getMessage ());
}
// clean up
$sqlArray [] = " DROP TABLE testtable; " ;
foreach ( $sqlArray as $i => $sql ) {
$this -> query ( $sql );
}
}
public function failingPreUpdate () {
throw new Exception ( 'Yolo fail' );
}
2019-09-30 15:31:49 +02:00
public function runUpdates ( $verbose = false , $useWorker = true , $processId = false )
2018-07-19 11:48:22 +02:00
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
2019-09-30 16:28:52 +02:00
$this -> Job = ClassRegistry :: init ( 'Job' );
$this -> Log = ClassRegistry :: init ( 'Log' );
2019-10-01 13:36:37 +02:00
$this -> Server = ClassRegistry :: init ( 'Server' );
2018-07-19 11:48:22 +02:00
$db = ConnectionManager :: getDataSource ( 'default' );
$tables = $db -> listSources ();
$requiresLogout = false ;
// if we don't even have an admin table, time to create it.
if ( ! in_array ( 'admin_settings' , $tables )) {
$this -> updateDatabase ( 'adminTable' );
$requiresLogout = true ;
} else {
$this -> __runCleanDB ();
$db_version = $this -> AdminSetting -> find ( 'all' , array ( 'conditions' => array ( 'setting' => 'db_version' )));
if ( count ( $db_version ) > 1 ) {
// we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
foreach ( $db_version as $k => $v ) {
if ( $k > 0 ) {
$this -> AdminSetting -> delete ( $v [ 'AdminSetting' ][ 'id' ]);
}
}
}
$db_version = $db_version [ 0 ];
$updates = $this -> __findUpgrades ( $db_version [ 'AdminSetting' ][ 'value' ]);
2019-10-01 13:08:09 +02:00
$job = $this -> Job -> find ( 'first' , array (
'conditions' => array ( 'Job.id' => $processId )
));
2020-01-20 10:25:00 +01:00
if ( ! empty ( $updates )) {
2019-09-30 16:28:52 +02:00
// Exit if updates are locked.
// This is not as reliable as a real lock implementation
// However, as all updates are re-playable, there is no harm if they
// get played multiple time. The purpose of this lightweight lock
// is only to limit the load.
if ( $this -> isUpdateLocked ()) { // prevent creation of useless workers
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
2019-11-18 11:35:10 +01:00
'action' => 'update_db_worker' ,
2019-09-30 16:28:52 +02:00
'user_id' => 0 ,
'title' => __ ( 'Issues executing run_updates' ),
2019-10-01 13:08:09 +02:00
'change' => __ ( 'Database updates are locked. Worker not spawned' )
2019-09-30 16:28:52 +02:00
));
2019-10-01 13:08:09 +02:00
if ( ! empty ( $job )) { // if multiple prio worker is enabled, want to mark them as done
$job [ 'Job' ][ 'progress' ] = 100 ;
$job [ 'Job' ][ 'message' ] = __ ( 'Update done' );
$this -> Job -> save ( $job );
}
return true ;
2019-09-30 16:28:52 +02:00
}
2019-10-01 13:08:09 +02:00
2019-05-08 15:37:39 +02:00
// restart this function by a worker
if ( $useWorker && Configure :: read ( 'MISP.background_jobs' )) {
2019-10-01 13:36:37 +02:00
$workerIssueCount = 0 ;
$workerDiagnostic = $this -> Server -> workerDiagnostics ( $workerIssueCount );
$workerType = '' ;
if ( isset ( $workerDiagnostic [ 'update' ][ 'ok' ]) && $workerDiagnostic [ 'update' ][ 'ok' ]) {
$workerType = 'update' ;
} elseif ( isset ( $workerDiagnostic [ 'prio' ][ 'ok' ]) && $workerDiagnostic [ 'prio' ][ 'ok' ]) {
$workerType = 'prio' ;
} else { // no worker running, doing inline update
return $this -> runUpdates ( $verbose , false );
}
2019-09-30 16:28:52 +02:00
$this -> Job -> create ();
2019-05-08 15:37:39 +02:00
$data = array (
2019-10-01 13:36:37 +02:00
'worker' => $workerType ,
2019-08-14 12:26:54 +02:00
'job_type' => 'run_updates' ,
2019-05-08 15:37:39 +02:00
'job_input' => 'command: ' . implode ( ',' , $updates ),
'status' => 0 ,
'retries' => 0 ,
'org_id' => 0 ,
'org' => '' ,
'message' => 'Updating.' ,
);
2019-09-30 16:28:52 +02:00
$this -> Job -> save ( $data );
$jobId = $this -> Job -> id ;
2019-10-29 09:57:25 +01:00
$processId = CakeResque :: enqueue (
2019-05-08 15:37:39 +02:00
'prio' ,
'AdminShell' ,
2019-08-14 12:26:54 +02:00
array ( 'runUpdates' , $jobId ),
2019-05-08 15:37:39 +02:00
true
);
2019-10-29 09:57:25 +01:00
$this -> Job -> saveField ( 'process_id' , $processId );
2019-05-08 15:37:39 +02:00
return true ;
}
2019-09-30 16:28:52 +02:00
// See comment above for `isUpdateLocked()`
2019-10-01 13:08:09 +02:00
// prevent continuation of job if worker was already spawned
// (could happens if multiple prio workers are up)
if ( $this -> isUpdateLocked ()) {
2019-09-30 16:28:52 +02:00
$this -> Log -> create ();
$this -> Log -> save ( array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
2019-11-18 11:35:10 +01:00
'action' => 'update_db_worker' ,
2019-09-30 16:28:52 +02:00
'user_id' => 0 ,
'title' => __ ( 'Issues executing run_updates' ),
2019-10-01 13:08:09 +02:00
'change' => __ ( 'Updates are locked. Stopping worker gracefully' )
2019-09-30 16:28:52 +02:00
));
2019-10-01 13:08:09 +02:00
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'progress' ] = 100 ;
$job [ 'Job' ][ 'message' ] = __ ( 'Update done' );
$this -> Job -> save ( $job );
}
return true ;
2019-09-30 16:28:52 +02:00
}
2019-10-10 12:02:23 +02:00
$this -> changeLockState ( time ());
2020-01-20 13:24:51 +01:00
$this -> __resetUpdateProgress ();
2019-09-30 16:28:52 +02:00
2019-09-30 15:31:49 +02:00
$update_done = 0 ;
2018-07-19 11:48:22 +02:00
foreach ( $updates as $update => $temp ) {
2019-02-10 21:35:23 +01:00
if ( $verbose ) {
echo str_pad ( 'Executing ' . $update , 30 , '.' );
}
2019-09-30 15:31:49 +02:00
if ( ! empty ( $job )) {
2019-10-01 13:08:09 +02:00
$job [ 'Job' ][ 'progress' ] = floor ( $update_done / count ( $updates ) * 100 );
2019-09-30 15:31:49 +02:00
$job [ 'Job' ][ 'message' ] = sprintf ( __ ( 'Running update %s' ), $update );
$this -> Job -> save ( $job );
}
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateMISP ( $update );
2018-07-19 11:48:22 +02:00
if ( $temp ) {
$requiresLogout = true ;
}
2019-10-29 09:57:25 +01:00
if ( $dbUpdateSuccess ) {
2019-08-14 12:13:45 +02:00
$db_version [ 'AdminSetting' ][ 'value' ] = $update ;
$this -> AdminSetting -> save ( $db_version );
2019-10-15 11:44:34 +02:00
$this -> resetUpdateFailNumber ();
2019-10-10 10:14:22 +02:00
} else {
$this -> __increaseUpdateFailNumber ();
2019-08-14 12:13:45 +02:00
}
2019-02-10 21:35:23 +01:00
if ( $verbose ) {
echo " \033 [32mDone \033 [0m " . PHP_EOL ;
}
2019-09-30 15:31:49 +02:00
$update_done ++ ;
}
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'message' ] = __ ( 'Update done' );
2018-07-19 11:48:22 +02:00
}
2019-10-10 12:02:23 +02:00
$this -> changeLockState ( false );
2018-07-19 11:48:22 +02:00
$this -> __queueCleanDB ();
2019-10-01 13:08:09 +02:00
} else {
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'message' ] = __ ( 'Update done in another worker. Gracefuly stopping.' );
}
}
// mark current worker as done, as well as queued workers than manages to pass the locks
// (happens if user hit reload before first worker start its job)
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'progress' ] = 100 ;
$this -> Job -> save ( $job );
2018-07-19 11:48:22 +02:00
}
}
if ( $requiresLogout ) {
$this -> updateDatabase ( 'destroyAllSessions' );
}
2019-04-26 09:45:03 +02:00
return true ;
}
2019-10-03 13:50:55 +02:00
private function __setUpdateProgress ( $current , $total = false , $toward_db_version = false )
2019-04-26 09:45:03 +02:00
{
$updateProgress = $this -> getUpdateProgress ();
2019-04-29 11:09:04 +02:00
$updateProgress [ 'current' ] = $current ;
2019-04-26 09:45:03 +02:00
if ( $total !== false ) {
2019-04-29 11:09:04 +02:00
$updateProgress [ 'total' ] = $total ;
2019-04-26 09:45:03 +02:00
} else {
$now = new DateTime ();
$updateProgress [ 'time' ][ 'started' ][ $current ] = $now -> format ( 'Y-m-d H:i:s' );
}
2019-10-03 13:50:55 +02:00
if ( $toward_db_version !== false ) {
$updateProgress [ 'toward_db_version' ] = $toward_db_version ;
}
2019-04-26 09:45:03 +02:00
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setPreUpdateTestState ( $state )
{
$updateProgress = $this -> getUpdateProgress ();
$updateProgress [ 'preTestSuccess' ] = $state ;
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setUpdateError ( $index )
{
$updateProgress = $this -> getUpdateProgress ();
$updateProgress [ 'failed_num' ][] = $index ;
$this -> __saveUpdateProgress ( $updateProgress );
}
2020-01-20 13:24:51 +01:00
private function __getEmptyUpdateMessage ()
2019-04-26 09:45:03 +02:00
{
2020-01-20 13:24:51 +01:00
return array (
2019-04-29 11:09:04 +02:00
'commands' => array (),
'results' => array (),
2019-04-26 09:45:03 +02:00
'time' => array ( 'started' => array (), 'elapsed' => array ()),
2019-04-29 11:09:04 +02:00
'current' => '' ,
'total' => '' ,
2019-10-03 13:50:55 +02:00
'failed_num' => array (),
'toward_db_version' => ''
2019-04-26 09:45:03 +02:00
);
2020-01-20 13:24:51 +01:00
}
private function __resetUpdateProgress ()
{
$updateProgress = $this -> __getEmptyUpdateMessage ();
2019-04-26 09:45:03 +02:00
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setUpdateCmdMessages ( $messages )
{
$updateProgress = $this -> getUpdateProgress ();
2019-04-29 11:09:04 +02:00
$updateProgress [ 'commands' ] = $messages ;
2019-04-26 09:45:03 +02:00
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setUpdateResMessages ( $index , $message )
{
$updateProgress = $this -> getUpdateProgress ();
2019-04-29 11:09:04 +02:00
$updateProgress [ 'results' ][ $index ] = $message ;
2019-04-26 09:45:03 +02:00
$temp = new DateTime ();
$diff = $temp -> diff ( new DateTime ( $updateProgress [ 'time' ][ 'started' ][ $index ]));
$updateProgress [ 'time' ][ 'elapsed' ][ $index ] = $diff -> format ( '%H:%I:%S' );
$this -> __saveUpdateProgress ( $updateProgress );
}
public function getUpdateProgress ()
{
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
$updateProgress = $this -> AdminSetting -> getSetting ( 'update_progress' );
if ( $updateProgress !== false ) {
$updateProgress = json_decode ( $updateProgress , true );
} else {
2020-01-20 13:24:51 +01:00
$updateProgress = $this -> __getEmptyUpdateMessage ();
2019-04-26 09:45:03 +02:00
}
foreach ( $updateProgress as $setting => $value ) {
if ( ! is_array ( $value )) {
2019-10-14 10:49:41 +02:00
if ( is_numeric ( $value )) {
$value = intval ( $value );
}
2019-04-26 09:45:03 +02:00
}
$updateProgress [ $setting ] = $value ;
}
return $updateProgress ;
}
private function __saveUpdateProgress ( $updateProgress )
{
2019-11-27 09:47:25 +01:00
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
2019-04-26 09:45:03 +02:00
$data = json_encode ( $updateProgress );
$this -> AdminSetting -> changeSetting ( 'update_progress' , $data );
}
2019-10-10 12:02:23 +02:00
public function changeLockState ( $locked )
2019-04-26 09:45:03 +02:00
{
2019-11-27 09:47:25 +01:00
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
2019-04-26 09:45:03 +02:00
$this -> AdminSetting -> changeSetting ( 'update_locked' , $locked );
}
private function getUpdateLockState ()
{
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
$locked = $this -> AdminSetting -> getSetting ( 'update_locked' );
return is_null ( $locked ) ? false : $locked ;
}
2019-10-01 15:22:29 +02:00
public function getLockRemainingTime ()
2019-04-26 09:45:03 +02:00
{
2019-12-09 09:53:18 +01:00
$lockState = $this -> getUpdateLockState ();
2019-04-29 11:09:04 +02:00
if ( $lockState !== false && $lockState !== '' ) {
2019-04-26 09:45:03 +02:00
// if lock is old, still allows the update
// This can be useful if the update process crashes
2019-04-29 11:09:04 +02:00
$diffSec = time () - intval ( $lockState );
2019-04-26 09:45:03 +02:00
if ( Configure :: read ( 'MISP.updateTimeThreshold' )) {
$updateWaitThreshold = intval ( Configure :: read ( 'MISP.updateTimeThreshold' ));
} else {
2019-04-29 11:09:04 +02:00
$this -> Server = ClassRegistry :: init ( 'Server' );
2019-04-26 09:45:03 +02:00
$updateWaitThreshold = intval ( $this -> Server -> serverSettings [ 'MISP' ][ 'updateTimeThreshold' ][ 'value' ]);
}
2019-10-01 15:22:29 +02:00
$remainingTime = $updateWaitThreshold - $diffSec ;
return $remainingTime > 0 ? $remainingTime : 0 ;
} else {
return 0 ;
2019-04-26 09:45:03 +02:00
}
2019-10-01 15:22:29 +02:00
}
public function isUpdateLocked ()
{
$remainingTime = $this -> getLockRemainingTime ();
2019-10-10 12:02:23 +02:00
$failThresholdReached = $this -> UpdateFailNumberReached ();
return $remainingTime > 0 || $failThresholdReached ;
2018-07-19 11:48:22 +02:00
}
2019-10-10 10:14:22 +02:00
public function getUpdateFailNumber ()
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
2019-10-29 09:57:25 +01:00
$updateFailNumber = $this -> AdminSetting -> getSetting ( 'update_fail_number' );
return ( $updateFailNumber !== false && $updateFailNumber !== '' ) ? $updateFailNumber : 0 ;
2019-10-10 10:14:22 +02:00
}
2019-10-15 11:44:34 +02:00
public function resetUpdateFailNumber ()
2019-10-10 10:14:22 +02:00
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$this -> AdminSetting -> changeSetting ( 'update_fail_number' , 0 );
}
2019-10-10 12:02:23 +02:00
public function __increaseUpdateFailNumber ()
2019-10-10 10:14:22 +02:00
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
2019-10-29 09:57:25 +01:00
$updateFailNumber = $this -> AdminSetting -> getSetting ( 'update_fail_number' );
$this -> AdminSetting -> changeSetting ( 'update_fail_number' , $updateFailNumber + 1 );
2019-10-10 10:14:22 +02:00
}
2019-10-10 12:02:23 +02:00
public function UpdateFailNumberReached ()
{
return $this -> getUpdateFailNumber () > 3 ;
}
2018-07-19 11:48:22 +02:00
private function __queueCleanDB ()
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$cleanDB = $this -> AdminSetting -> find ( 'first' , array ( 'conditions' => array ( 'setting' => 'clean_db' )));
if ( empty ( $cleanDB )) {
$this -> AdminSetting -> create ();
$cleanDB = array ( 'AdminSetting' => array ( 'setting' => 'clean_db' , 'value' => 1 ));
} else {
$cleanDB [ 'AdminSetting' ][ 'value' ] = 1 ;
}
$this -> AdminSetting -> save ( $cleanDB );
}
private function __runCleanDB ()
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$cleanDB = $this -> AdminSetting -> find ( 'first' , array ( 'conditions' => array ( 'setting' => 'clean_db' )));
if ( empty ( $cleanDB ) || $cleanDB [ 'AdminSetting' ][ 'value' ] == 1 ) {
$this -> cleanCacheFiles ();
if ( empty ( $cleanDB )) {
$this -> AdminSetting -> create ();
$cleanDB = array ( 'AdminSetting' => array ( 'setting' => 'clean_db' , 'value' => 0 ));
} else {
$cleanDB [ 'AdminSetting' ][ 'value' ] = 0 ;
}
$this -> AdminSetting -> save ( $cleanDB );
}
}
private function __findUpgrades ( $db_version )
{
$updates = array ();
if ( strpos ( $db_version , '.' )) {
$version = explode ( '.' , $db_version );
foreach ( $this -> old_db_changes as $major => $rest ) {
if ( $major < $version [ 0 ]) {
continue ;
} elseif ( $major == $version [ 0 ]) {
foreach ( $rest as $minor => $hotfixes ) {
if ( $minor < $version [ 1 ]) {
continue ;
} elseif ( $minor == $version [ 1 ]) {
foreach ( $hotfixes as $hotfix => $requiresLogout ) {
if ( $hotfix > $version [ 2 ]) {
$updates [ $major . '.' . $minor . '.' . $hotfix ] = $requiresLogout ;
}
}
} else {
foreach ( $hotfixes as $hotfix => $requiresLogout ) {
$updates [ $major . '.' . $minor . '.' . $hotfix ] = $requiresLogout ;
}
}
}
}
}
$db_version = 0 ;
}
foreach ( $this -> db_changes as $db_change => $requiresLogout ) {
if ( $db_version < $db_change ) {
$updates [ $db_change ] = $requiresLogout ;
}
}
return $updates ;
}
2020-02-26 14:37:09 +01:00
private function __generateCorrelations ()
{
if ( Configure :: read ( 'MISP.background_jobs' )) {
$Job = ClassRegistry :: init ( 'Job' );
$Job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'generate correlation' ,
'job_input' => 'All attributes' ,
'status' => 0 ,
'retries' => 0 ,
'org' => 'ADMIN' ,
'message' => 'Job created.' ,
);
$Job -> save ( $data );
$jobId = $Job -> id ;
$process_id = CakeResque :: enqueue (
'default' ,
'AdminShell' ,
array ( 'jobGenerateCorrelation' , $jobId ),
true
);
$Job -> saveField ( 'process_id' , $process_id );
}
return true ;
}
2019-09-30 08:23:36 +02:00
public function populateNotifications ( $user , $mode = 'full' )
2018-07-19 11:48:22 +02:00
{
$notifications = array ();
2019-09-30 08:23:36 +02:00
list ( $notifications [ 'proposalCount' ], $notifications [ 'proposalEventCount' ]) = $this -> _getProposalCount ( $user , $mode );
$notifications [ 'total' ] = $notifications [ 'proposalCount' ];
2018-07-19 11:48:22 +02:00
if ( Configure :: read ( 'MISP.delegation' )) {
2019-09-30 08:23:36 +02:00
$notifications [ 'delegationCount' ] = $this -> _getDelegationCount ( $user );
$notifications [ 'total' ] += $notifications [ 'delegationCount' ];
2018-07-19 11:48:22 +02:00
}
return $notifications ;
}
2019-09-30 08:23:36 +02:00
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount ( $user , $mode = 'full' )
2018-07-19 11:48:22 +02:00
{
$this -> ShadowAttribute = ClassRegistry :: init ( 'ShadowAttribute' );
2019-09-30 08:23:36 +02:00
$results [ 0 ] = $this -> ShadowAttribute -> find (
'count' ,
array (
2018-07-19 11:48:22 +02:00
'recursive' => - 1 ,
'conditions' => array (
'ShadowAttribute.event_org_id' => $user [ 'org_id' ],
'ShadowAttribute.deleted' => 0 ,
2019-09-30 08:23:36 +02:00
)
)
);
if ( $mode === 'full' ) {
$results [ 1 ] = $this -> ShadowAttribute -> find (
'count' ,
array (
'recursive' => - 1 ,
'conditions' => array (
'ShadowAttribute.event_org_id' => $user [ 'org_id' ],
'ShadowAttribute.deleted' => 0 ,
),
'fields' => 'distinct event_id'
)
);
} else {
$results [ 1 ] = $results [ 0 ];
2018-07-19 11:48:22 +02:00
}
return $results ;
}
private function _getDelegationCount ( $user )
{
$this -> EventDelegation = ClassRegistry :: init ( 'EventDelegation' );
$delegations = $this -> EventDelegation -> find ( 'count' , array (
2019-09-30 08:23:36 +02:00
'recursive' => - 1 ,
'conditions' => array ( 'EventDelegation.org_id' => $user [ 'org_id' ])
2018-07-19 11:48:22 +02:00
));
return $delegations ;
}
public function checkFilename ( $filename )
{
return preg_match ( '@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i' , $filename );
}
2019-09-20 22:15:15 +02:00
/**
* Similar method as `setupRedis` , but this method throw exception if Redis cannot be reached .
* @ return Redis
* @ throws Exception
*/
public function setupRedisWithException ()
2018-07-19 11:48:22 +02:00
{
2019-08-16 19:26:12 +02:00
if ( $this -> __redisConnection ) {
return $this -> __redisConnection ;
}
if ( ! class_exists ( 'Redis' )) {
2019-09-20 22:15:15 +02:00
throw new Exception ( " Class Redis doesn't exists. " );
2018-07-19 11:48:22 +02:00
}
2019-08-16 19:26:12 +02:00
$host = Configure :: read ( 'MISP.redis_host' ) ? : '127.0.0.1' ;
$port = Configure :: read ( 'MISP.redis_port' ) ? : 6379 ;
$database = Configure :: read ( 'MISP.redis_database' ) ? : 13 ;
2018-07-19 11:48:22 +02:00
$pass = Configure :: read ( 'MISP.redis_password' );
2019-08-16 19:26:12 +02:00
$redis = new Redis ();
2018-07-19 11:48:22 +02:00
if ( ! $redis -> connect ( $host , $port )) {
2019-09-20 22:15:15 +02:00
throw new Exception ( " Could not connect to Redis: { $redis -> getLastError () } " );
2018-07-19 11:48:22 +02:00
}
if ( ! empty ( $pass )) {
2019-09-20 22:15:15 +02:00
if ( ! $redis -> auth ( $pass )) {
throw new Exception ( " Could not authenticate to Redis: { $redis -> getLastError () } " );
}
2018-07-19 11:48:22 +02:00
}
2019-09-20 22:15:15 +02:00
if ( ! $redis -> select ( $database )) {
throw new Exception ( " Could not select Redis database $database : { $redis -> getLastError () } " );
2018-07-19 11:48:22 +02:00
}
2019-09-20 22:15:15 +02:00
2018-07-19 11:48:22 +02:00
$this -> __redisConnection = $redis ;
return $redis ;
}
2019-09-20 22:15:15 +02:00
/**
* Method for backward compatibility .
* @ deprecated
* @ see AppModel :: setupRedisWithException
* @ return bool | Redis
*/
public function setupRedis ()
{
try {
return $this -> setupRedisWithException ();
} catch ( Exception $e ) {
return false ;
}
}
2019-03-05 12:24:56 +01:00
public function getKafkaPubTool ()
{
if ( ! $this -> loadedKafkaPubTool ) {
$this -> loadKafkaPubTool ();
}
return $this -> loadedKafkaPubTool ;
}
public function loadKafkaPubTool ()
{
App :: uses ( 'KafkaPubTool' , 'Tools' );
$kafkaPubTool = new KafkaPubTool ();
$rdkafkaIni = Configure :: read ( 'Plugin.Kafka_rdkafka_config' );
$kafkaConf = array ();
if ( ! empty ( $rdkafkaIni )) {
$kafkaConf = parse_ini_file ( $rdkafkaIni );
}
$brokers = Configure :: read ( 'Plugin.Kafka_brokers' );
$kafkaPubTool -> initTool ( $brokers , $kafkaConf );
$this -> loadedKafkaPubTool = $kafkaPubTool ;
return true ;
}
public function publishKafkaNotification ( $topicName , $data , $action = false ) {
$kafkaTopic = Configure :: read ( 'Plugin.Kafka_' . $topicName . '_notifications_topic' );
if ( Configure :: read ( 'Plugin.Kafka_enable' ) && Configure :: read ( 'Plugin.Kafka_' . $topicName . '_notifications_enable' ) && ! empty ( $kafkaTopic )) {
$this -> getKafkaPubTool () -> publishJson ( $kafkaTopic , $data , $action );
}
}
2018-07-19 11:48:22 +02:00
public function getPubSubTool ()
{
if ( ! $this -> loadedPubSubTool ) {
2020-05-07 15:50:54 +02:00
App :: uses ( 'PubSubTool' , 'Tools' );
$pubSubTool = new PubSubTool ();
$pubSubTool -> initTool ();
$this -> loadedPubSubTool = $pubSubTool ;
2018-07-19 11:48:22 +02:00
}
return $this -> loadedPubSubTool ;
}
public function getElasticSearchTool ()
{
if ( ! $this -> elasticSearchClient ) {
$this -> loadElasticSearchTool ();
}
return $this -> elasticSearchClient ;
}
public function loadElasticSearchTool ()
{
App :: uses ( 'ElasticSearchClient' , 'Tools' );
$client = new ElasticSearchClient ();
$client -> initTool ();
$this -> elasticSearchClient = $client ;
}
public function checkVersionRequirements ( $versionString , $minVersion )
{
$version = explode ( '.' , $versionString );
$minVersion = explode ( '.' , $minVersion );
if ( count ( $version ) > $minVersion ) {
return true ;
}
if ( count ( $version ) == 1 ) {
return $minVersion <= $version ;
}
return ( $version [ 0 ] >= $minVersion [ 0 ] && $version [ 1 ] >= $minVersion [ 1 ] && $version [ 2 ] >= $minVersion [ 2 ]);
}
// generate a generic subquery - options needs to include conditions
2018-08-02 15:40:53 +02:00
public function subQueryGenerator ( $model , $options , $lookupKey , $negation = false )
2018-07-19 11:48:22 +02:00
{
$db = $model -> getDataSource ();
$defaults = array (
'fields' => array ( '*' ),
2018-08-02 15:40:53 +02:00
'table' => $model -> table ,
2018-07-19 11:48:22 +02:00
'alias' => $model -> alias ,
'limit' => null ,
'offset' => null ,
'joins' => array (),
'conditions' => array (),
2020-06-14 20:23:48 +02:00
'group' => false ,
'recursive' => - 1
2018-07-19 11:48:22 +02:00
);
$params = array ();
foreach ( array_keys ( $defaults ) as $key ) {
2018-08-02 15:40:53 +02:00
if ( isset ( $options [ $key ])) {
$params [ $key ] = $options [ $key ];
2018-07-19 11:48:22 +02:00
} else {
2018-08-02 15:40:53 +02:00
$params [ $key ] = $defaults [ $key ];
2018-07-19 11:48:22 +02:00
}
}
$subQuery = $db -> buildStatement (
$params ,
$model
);
2018-08-02 15:40:53 +02:00
if ( $negation ) {
$subQuery = $lookupKey . ' NOT IN (' . $subQuery . ') ' ;
} else {
$subQuery = $lookupKey . ' IN (' . $subQuery . ') ' ;
}
2018-07-19 11:48:22 +02:00
$conditions = array (
$db -> expression ( $subQuery ) -> value
);
return $conditions ;
}
// start a benchmark run for the given bench name
public function benchmarkInit ( $name = 'default' )
{
$this -> __profiler [ $name ][ 'start' ] = microtime ( true );
if ( empty ( $this -> __profiler [ $name ][ 'memory_start' ])) {
$this -> __profiler [ $name ][ 'memory_start' ] = memory_get_usage ();
}
return true ;
}
// calculate the duration from the init time to the current point in execution. Aggregate flagged executions will increment the duration instead of just setting it
public function benchmark ( $name = 'default' , $aggregate = false , $memory_chart = false )
{
if ( ! empty ( $this -> __profiler [ $name ][ 'start' ])) {
if ( $aggregate ) {
if ( ! isset ( $this -> __profiler [ $name ][ 'duration' ])) {
$this -> __profiler [ $name ][ 'duration' ] = 0 ;
}
if ( ! isset ( $this -> __profiler [ $name ][ 'executions' ])) {
$this -> __profiler [ $name ][ 'executions' ] = 0 ;
}
$this -> __profiler [ $name ][ 'duration' ] += microtime ( true ) - $this -> __profiler [ $name ][ 'start' ];
$this -> __profiler [ $name ][ 'executions' ] ++ ;
$currentUsage = memory_get_usage ();
if ( $memory_chart ) {
$this -> __profiler [ $name ][ 'memory_chart' ][] = $currentUsage - $this -> __profiler [ $name ][ 'memory_start' ];
}
if (
empty ( $this -> __profiler [ $name ][ 'memory_peak' ]) ||
$this -> __profiler [ $name ][ 'memory_peak' ] < ( $currentUsage - $this -> __profiler [ $name ][ 'memory_start' ])
) {
$this -> __profiler [ $name ][ 'memory_peak' ] = $currentUsage - $this -> __profiler [ $name ][ 'memory_start' ];
}
} else {
$this -> __profiler [ $name ][ 'memory_peak' ] = memory_get_usage () - $this -> __profiler [ $name ][ 'memory_start' ];
$this -> __profiler [ $name ][ 'duration' ] = microtime ( true ) - $this -> __profiler [ $name ][ 'start' ];
}
}
return true ;
}
// return the results of the benchmark(s). If no name is set all benchmark results are returned in an array.
public function benchmarkResult ( $name = false )
{
if ( $name ) {
return array ( $name => $this -> __profiler [ $name ][ 'duration' ]);
} else {
$results = array ();
foreach ( $this -> __profiler as $name => $benchmark ) {
if ( ! empty ( $benchmark [ 'duration' ])) {
$results [ $name ] = $benchmark ;
unset ( $results [ $name ][ 'start' ]);
unset ( $results [ $name ][ 'memory_start' ]);
}
}
return $results ;
}
}
public function getRowCount ( $table = false )
{
if ( empty ( $table )) {
$table = $this -> table ;
}
$table_data = $this -> query ( " show table status like ' " . $table . " ' " );
return $table_data [ 0 ][ 'TABLES' ][ 'Rows' ];
}
public function benchmarkCustomAdd ( $valueToAdd = 0 , $name = 'default' , $customName = 'custom' )
{
if ( empty ( $this -> __profiler [ $name ][ 'custom' ][ $customName ])) {
$this -> __profiler [ $name ][ 'custom' ][ $customName ] = 0 ;
}
$this -> __profiler [ $name ][ 'custom' ][ $customName ] += $valueToAdd ;
}
private function __forceSettings ()
{
$settingsToForce = array (
'Session.autoRegenerate' => false ,
'Session.checkAgent' => false
);
$server = ClassRegistry :: init ( 'Server' );
foreach ( $settingsToForce as $setting => $value ) {
$server -> serverSettingsSaveValue ( $setting , $value );
}
return true ;
}
2018-08-03 11:52:20 +02:00
2020-03-02 23:09:47 +01:00
public function setupHttpSocket ( $server , $HttpSocket = null , $timeout = false )
2018-08-03 11:52:20 +02:00
{
if ( empty ( $HttpSocket )) {
App :: uses ( 'SyncTool' , 'Tools' );
$syncTool = new SyncTool ();
2020-03-02 23:09:47 +01:00
$HttpSocket = $syncTool -> setupHttpSocket ( $server , $timeout );
2018-08-03 11:52:20 +02:00
}
return $HttpSocket ;
}
public function setupSyncRequest ( $server )
{
$request = array (
'header' => array (
'Authorization' => $server [ 'Server' ][ 'authkey' ],
'Accept' => 'application/json' ,
'Content-Type' => 'application/json'
)
);
$request = $this -> addHeaders ( $request );
return $request ;
}
2018-08-03 13:22:17 +02:00
public function addHeaders ( $request )
{
$version = $this -> checkMISPVersion ();
$version = implode ( '.' , $version );
try {
$commit = trim ( shell_exec ( 'git log --pretty="%H" -n1 HEAD' ));
} catch ( Exception $e ) {
$commit = false ;
}
$request [ 'header' ][ 'MISP-version' ] = $version ;
if ( $commit ) {
$request [ 'header' ][ 'commit' ] = $commit ;
}
return $request ;
}
2018-08-09 07:44:53 +02:00
// take filters in the {"OR" => [foo], "NOT" => [bar]} format along with conditions and set the conditions
2018-10-08 10:17:57 +02:00
public function generic_add_filter ( $conditions , & $filter , $keys )
2018-08-09 07:44:53 +02:00
{
$operator_composition = array (
'NOT' => 'AND' ,
'OR' => 'OR' ,
'AND' => 'AND'
);
if ( ! is_array ( $keys )) {
$keys = array ( $keys );
}
2018-09-03 17:54:37 +02:00
if ( ! isset ( $filter [ 'OR' ]) && ! isset ( $filter [ 'AND' ]) && ! isset ( $filter [ 'NOT' ])) {
2018-08-09 07:44:53 +02:00
return $conditions ;
}
foreach ( $filter as $operator => $filters ) {
$temp = array ();
2018-11-23 14:11:33 +01:00
if ( ! is_array ( $filters )) {
$filters = array ( $filters );
}
2018-08-09 07:44:53 +02:00
foreach ( $filters as $f ) {
2018-11-23 14:11:33 +01:00
if ( $f === - 1 ) {
foreach ( $keys as $key ) {
$temp [ 'OR' ][ $key ][] = - 1 ;
}
continue ;
}
2018-08-09 07:44:53 +02:00
// split the filter params into two lists, one for substring searches one for exact ones
2018-10-11 20:20:27 +02:00
if ( is_string ( $f ) && ( $f [ strlen ( $f ) - 1 ] === '%' || $f [ 0 ] === '%' )) {
2018-08-09 07:44:53 +02:00
foreach ( $keys as $key ) {
if ( $operator === 'NOT' ) {
$temp [] = array ( $key . ' NOT LIKE' => $f );
} else {
$temp [] = array ( $key . ' LIKE' => $f );
}
}
} else {
foreach ( $keys as $key ) {
if ( $operator === 'NOT' ) {
$temp [ $key . ' !=' ][] = $f ;
} else {
2018-08-09 15:11:57 +02:00
$temp [ 'OR' ][ $key ][] = $f ;
2018-08-09 07:44:53 +02:00
}
}
}
}
2018-11-23 14:11:33 +01:00
$conditions [ 'AND' ][] = array ( $operator_composition [ $operator ] => $temp );
2018-08-09 07:44:53 +02:00
if ( $operator !== 'NOT' ) {
unset ( $filter [ $operator ]);
}
}
return $conditions ;
}
/*
* Get filters in one of the following formats :
* [ foo , bar ]
* [ " OR " => [ foo , bar ], " NOT " => [ baz ]]
* " foo "
* " foo&&bar&&!baz "
* and convert it into the same format [ " OR " => [ foo , bar ], " NOT " => [ baz ]]
*/
public function convert_filters ( $filter )
{
if ( ! is_array ( $filter )) {
$temp = explode ( '&&' , $filter );
$filter = array ();
foreach ( $temp as $f ) {
if ( $f [ 0 ] === '!' ) {
2019-04-23 14:02:27 +02:00
$filter [ 'NOT' ][] = substr ( $f , 1 );
2018-08-09 07:44:53 +02:00
} else {
$filter [ 'OR' ][] = $f ;
}
}
return $filter ;
}
if ( ! isset ( $filter [ 'OR' ]) && ! isset ( $filter [ 'NOT' ]) && ! isset ( $filter [ 'AND' ])) {
$temp = array ();
foreach ( $filter as $param ) {
2018-11-23 14:11:33 +01:00
if ( ! empty ( $param )) {
if ( $param [ 0 ] === '!' ) {
$temp [ 'NOT' ][] = substr ( $param , 1 );
} else {
$temp [ 'OR' ][] = $param ;
}
}
2018-08-09 07:44:53 +02:00
}
$filter = $temp ;
}
return $filter ;
}
2018-09-28 09:21:29 +02:00
2018-11-23 14:11:33 +01:00
public function convert_to_memory_limit_to_mb ( $val )
{
$val = trim ( $val );
if ( $val == - 1 ) {
// default to 8GB if no limit is set
return 8 * 1024 ;
}
$unit = $val [ strlen ( $val ) - 1 ];
if ( is_numeric ( $unit )) {
$unit = 'b' ;
} else {
$val = intval ( $val );
}
$unit = strtolower ( $unit );
switch ( $unit ) {
case 'g' :
$val *= 1024 ;
// no break
case 'm' :
$val *= 1024 ;
// no break
case 'k' :
$val *= 1024 ;
}
return $val / ( 1024 * 1024 );
}
public function getDefaultAttachments_dir ()
{
return APP . 'files' ;
}
public function getDefaultTmp_dir ()
{
return sys_get_temp_dir ();
}
private function __bumpReferences ()
{
$this -> Event = ClassRegistry :: init ( 'Event' );
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$existingSetting = $this -> AdminSetting -> find ( 'first' , array (
'conditions' => array ( 'AdminSetting.setting' => 'update_23' )
));
if ( empty ( $existingSetting )) {
$this -> AdminSetting -> create ();
$data = array (
'setting' => 'update_23' ,
'value' => 1
);
$this -> AdminSetting -> save ( $data );
$references = $this -> Event -> Object -> ObjectReference -> find ( 'list' , array (
'recursive' => - 1 ,
'fields' => array ( 'ObjectReference.event_id' , 'ObjectReference.event_id' ),
'group' => array ( 'ObjectReference.event_id' )
));
$event_ids = array ();
$object_ids = array ();
foreach ( $references as $reference ) {
$event = $this -> Event -> find ( 'first' , array (
'conditions' => array (
'Event.id' => $reference ,
'Event.locked' => 0
),
'recursive' => - 1 ,
'fields' => array ( 'Event.id' , 'Event.locked' )
));
if ( ! empty ( $event )) {
$event_ids [] = $event [ 'Event' ][ 'id' ];
$event_references = $this -> Event -> Object -> ObjectReference -> find ( 'list' , array (
'conditions' => array ( 'ObjectReference.event_id' => $reference ),
'recursive' => - 1 ,
'fields' => array ( 'ObjectReference.object_id' , 'ObjectReference.object_id' )
));
$object_ids = array_merge ( $object_ids , array_values ( $event_references ));
}
}
if ( ! empty ( $object_ids )) {
$this -> Event -> Object -> updateAll (
array (
'Object.timestamp' => 'Object.timestamp + 1'
),
array ( 'Object.id' => $object_ids )
);
$this -> Event -> updateAll (
array (
'Event.timestamp' => 'Event.timestamp + 1'
),
array ( 'Event.id' => $event_ids )
);
}
$this -> Log = ClassRegistry :: init ( 'Log' );
$this -> Log -> create ();
$entry = array (
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Bumped the timestamps of locked events containing object references.' ,
'change' => sprintf ( 'Event timestamps updated: %s; Object timestamps updated: %s' , count ( $event_ids ), count ( $object_ids ))
);
$this -> Log -> save ( $entry );
}
return true ;
}
2018-11-26 09:12:01 +01:00
2019-02-10 13:08:12 +01:00
public function generateRandomFileName ()
{
return ( new RandomTool ()) -> random_str ( false , 12 );
}
2019-02-06 17:46:36 +01:00
public function resolveTimeDelta ( $delta )
{
if ( is_numeric ( $delta )) {
return $delta ;
}
$multiplierArray = array ( 'd' => 86400 , 'h' => 3600 , 'm' => 60 , 's' => 1 );
$multiplier = $multiplierArray [ 'd' ];
$lastChar = strtolower ( substr ( $delta , - 1 ));
if ( ! is_numeric ( $lastChar ) && array_key_exists ( $lastChar , $multiplierArray )) {
$multiplier = $multiplierArray [ $lastChar ];
$delta = substr ( $delta , 0 , - 1 );
} else if ( strtotime ( $delta ) !== false ) {
return strtotime ( $delta );
} else {
// invalid filter, make sure we don't return anything
return time () + 1 ;
}
if ( ! is_numeric ( $delta )) {
// Same here. (returning false dumps the whole database)
return time () + 1 ;
}
return time () - ( $delta * $multiplier );
}
2019-03-25 17:35:02 +01:00
private function __fixServerPullPushRules ()
{
$this -> Server = ClassRegistry :: init ( 'Server' );
$servers = $this -> Server -> find ( 'all' , array ( 'recursive' => - 1 ));
foreach ( $servers as $server ) {
$changed = false ;
if ( empty ( $server [ 'Server' ][ 'pull_rules' ])) {
$server [ 'Server' ][ 'pull_rules' ] = '[]' ;
$changed = true ;
}
if ( empty ( $server [ 'Server' ][ 'push_rules' ])) {
$server [ 'Server' ][ 'push_rules' ] = '[]' ;
$changed = true ;
}
if ( $changed ) {
$this -> Server -> save ( $server );
}
}
}
2019-10-08 11:43:56 +02:00
2019-09-24 20:52:36 +02:00
/**
2020-04-26 21:44:22 +02:00
* Log exception with backtrace and with nested exceptions .
*
2019-09-24 20:52:36 +02:00
* @ param string $message
* @ param Exception $exception
* @ param int $type
* @ return bool
*/
protected function logException ( $message , Exception $exception , $type = LOG_ERR )
{
2020-01-26 18:17:49 +01:00
$message .= " \n " ;
do {
$message .= sprintf ( " [%s] %s " ,
get_class ( $exception ),
$exception -> getMessage ()
);
$message .= " \n Stack Trace: \n " . $exception -> getTraceAsString ();
$exception = $exception -> getPrevious ();
} while ( $exception !== null );
2019-09-24 20:52:36 +02:00
return $this -> log ( $message , $type );
}
2020-04-28 15:31:27 +02:00
/**
* Generates random file name in tmp dir .
* @ return string
*/
protected function tempFileName ()
{
return $this -> tempDir () . DS . $this -> generateRandomFileName ();
}
/**
* @ return string
*/
protected function tempDir ()
{
return Configure :: read ( 'MISP.tmpdir' ) ? : sys_get_temp_dir ();
}
2020-06-19 10:22:19 +02:00
/**
* Decodes JSON string and throws exception if string is not valid JSON or if is not array .
*
* @ param string $json
* @ return array
* @ throws JsonException
* @ throws UnexpectedValueException
*/
2020-07-27 12:10:53 +02:00
public function jsonDecode ( $json )
2020-06-19 10:22:19 +02:00
{
if ( defined ( 'JSON_THROW_ON_ERROR' )) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
$decoded = json_decode ( $json , true , 512 , JSON_THROW_ON_ERROR );
} else {
$decoded = json_decode ( $json , true );
if ( $decoded === null ) {
throw new UnexpectedValueException ( 'Could not parse JSON: ' . json_last_error_msg (), json_last_error ());
}
}
if ( ! is_array ( $decoded )) {
throw new UnexpectedValueException ( 'JSON must be array type, get ' . gettype ( $decoded ));
}
return $decoded ;
}
2020-07-30 09:17:27 +02:00
/*
* Temporary solution for utf8 columns until we migrate to utf8mb4
* via https :// stackoverflow . com / questions / 16496554 / can - php - detect - 4 - byte - encoded - utf8 - chars
*/
public function handle4ByteUnicode ( $input )
{
return preg_replace (
' % ( ? :
\xF0 [ \x90 - \xBF ][ \x80 - \xBF ]{ 2 }
| [ \xF1 - \xF3 ][ \x80 - \xBF ]{ 3 }
| \xF4 [ \x80 - \x8F ][ \x80 - \xBF ]{ 2 }
) % xs ' ,
'?' ,
$input
);
}
2020-08-13 15:58:42 +02:00
/**
* @ return AttachmentTool
*/
protected function loadAttachmentTool ()
{
if ( $this -> attachmentTool === null ) {
$this -> attachmentTool = new AttachmentTool ();
}
return $this -> attachmentTool ;
}
2012-03-15 15:06:45 +01:00
}