2012-03-15 15:06:45 +01:00
< ? php
/**
* Application model for Cake .
*
* This file is application - wide model file . You can put all
* application - wide model - related methods here .
*
* PHP 5
*
* CakePHP ( tm ) : Rapid Development Framework ( http :// cakephp . org )
2012-09-25 15:41:58 +02:00
* Copyright 2005 - 2012 , Cake Software Foundation , Inc . ( http :// cakefoundation . org )
2012-03-15 15:06:45 +01:00
*
* Licensed under The MIT License
* Redistributions of files must retain the above copyright notice .
*
2012-09-25 15:41:58 +02:00
* @ copyright Copyright 2005 - 2012 , Cake Software Foundation , Inc . ( http :// cakefoundation . org )
2012-03-15 15:06:45 +01:00
* @ link http :// cakephp . org CakePHP ( tm ) Project
* @ package app . Model
* @ since CakePHP ( tm ) v 0.2 . 9
* @ license MIT License ( http :// www . opensource . org / licenses / mit - license . php )
*/
App :: uses ( 'Model' , 'Model' );
2012-09-20 11:34:41 +02:00
App :: uses ( 'LogableBehavior' , 'Assets.models/behaviors' );
2018-11-26 09:12:01 +01:00
App :: uses ( 'RandomTool' , 'Tools' );
2021-11-07 10:40:01 +01:00
App :: uses ( 'FileAccessTool' , 'Tools' );
App :: uses ( 'JsonTool' , 'Tools' );
2021-07-24 22:58:21 +02:00
2018-07-19 11:48:22 +02:00
class AppModel extends Model
{
2021-07-25 20:03:32 +02:00
/** @var PubSubTool */
private static $loadedPubSubTool ;
2018-07-19 11:48:22 +02:00
2021-07-24 22:58:21 +02:00
/** @var KafkaPubTool */
2021-07-25 20:03:32 +02:00
private $loadedKafkaPubTool ;
2018-07-19 11:48:22 +02:00
2021-10-21 13:39:26 +02:00
/** @var BackgroundJobsTool */
private static $loadedBackgroundJobsTool ;
2021-02-15 18:28:41 +01:00
/** @var null|Redis */
2021-07-25 20:03:32 +02:00
private static $__redisConnection ;
2018-07-19 11:48:22 +02:00
private $__profiler = array ();
2021-11-07 10:40:01 +01:00
public $elasticSearchClient ;
2018-07-19 11:48:22 +02:00
2020-08-13 15:58:42 +02:00
/** @var AttachmentTool|null */
private $attachmentTool ;
2018-07-19 11:48:22 +02:00
public function __construct ( $id = false , $table = null , $ds = null )
{
parent :: __construct ( $id , $table , $ds );
2020-12-22 23:43:17 +01:00
$this -> findMethods [ 'column' ] = true ;
2022-04-17 16:30:14 +02:00
if ( in_array ( 'phar' , stream_get_wrappers ())) {
stream_wrapper_unregister ( 'phar' );
}
2018-07-19 11:48:22 +02:00
}
// deprecated, use $db_changes
// major -> minor -> hotfix -> requires_logout
2021-07-27 18:49:40 +02:00
const OLD_DB_CHANGES = array (
2018-07-19 11:48:22 +02:00
2 => array (
4 => array (
18 => false , 19 => false , 20 => false , 25 => false , 27 => false ,
32 => false , 33 => true , 38 => true , 39 => true , 40 => false ,
42 => false , 44 => false , 45 => false , 49 => true , 50 => false ,
51 => false , 52 => false , 55 => true , 56 => true , 57 => true ,
58 => false , 59 => false , 60 => false , 61 => false , 62 => false ,
63 => false , 64 => false , 65 => false , 66 => false , 67 => true ,
68 => false , 69 => false , 71 => false , 72 => false , 73 => false ,
75 => false , 77 => false , 78 => false , 79 => false , 80 => false ,
81 => false , 82 => false , 83 => false , 84 => false , 85 => false ,
86 => false , 87 => false
)
)
);
2021-07-27 18:49:40 +02:00
const DB_CHANGES = array (
2018-07-19 11:48:22 +02:00
1 => false , 2 => false , 3 => false , 4 => true , 5 => false , 6 => false ,
7 => false , 8 => false , 9 => false , 10 => false , 11 => false , 12 => false ,
2018-09-12 19:17:32 +02:00
13 => false , 14 => false , 15 => false , 18 => false , 19 => false , 20 => false ,
2019-01-01 16:37:15 +01:00
21 => false , 22 => false , 23 => false , 24 => false , 25 => false , 26 => false ,
2019-03-19 17:23:05 +01:00
27 => false , 28 => false , 29 => false , 30 => false , 31 => false , 32 => false ,
2019-09-25 11:50:54 +02:00
33 => false , 34 => false , 35 => false , 36 => false , 37 => false , 38 => false ,
2019-11-22 21:53:51 +01:00
39 => false , 40 => false , 41 => false , 42 => false , 43 => false , 44 => false ,
2020-04-09 14:26:48 +02:00
45 => false , 46 => false , 47 => false , 48 => false , 49 => false , 50 => false ,
2020-07-07 08:12:12 +02:00
51 => false , 52 => false , 53 => false , 54 => false , 55 => false , 56 => false ,
2020-11-18 11:45:01 +01:00
57 => false , 58 => false , 59 => false , 60 => false , 61 => false , 62 => false ,
2021-01-22 13:01:23 +01:00
63 => true , 64 => false , 65 => false , 66 => false , 67 => false , 68 => false ,
2021-10-23 19:07:29 +02:00
69 => false , 70 => false , 71 => true , 72 => true , 73 => false , 74 => false ,
2022-03-02 17:32:35 +01:00
75 => false , 76 => true , 77 => false , 78 => false , 79 => false , 80 => false ,
2022-05-04 09:04:30 +02:00
81 => false , 82 => false , 83 => false , 84 => false , 85 => false , 86 => false
2018-07-19 11:48:22 +02:00
);
2019-04-26 09:45:03 +02:00
public $advanced_updates_description = array (
2019-06-13 09:16:34 +02:00
'seenOnAttributeAndObject' => array (
'title' => 'First seen/Last seen Attribute table' ,
'description' => 'Update the Attribute table to support first_seen and last_seen feature, with a microsecond resolution.' ,
'liveOff' => true , # should the instance be offline for users other than site_admin
'recommendBackup' => true , # should the update recommend backup
'exitOnError' => false , # should the update exit on error
'requirements' => 'MySQL version must be >= 5.6' , # message stating the requirements necessary for the update
2019-07-02 15:22:21 +02:00
'record' => false , # should the update success be saved in the admin_table
2019-07-01 16:27:09 +02:00
// 'preUpdate' => 'seenOnAttributeAndObjectPreUpdate', # Function to execute before the update. If it throws an error, it cancels the update
2019-06-13 09:16:34 +02:00
'url' => '/servers/updateDatabase/seenOnAttributeAndObject/' # url pointing to the funcion performing the update
),
2019-04-26 09:45:03 +02:00
);
2022-05-14 10:17:06 +02:00
public function isAcceptedDatabaseError ( $errorMessage )
2019-10-14 12:37:19 +02:00
{
2022-05-14 10:17:06 +02:00
if ( $this -> isMysql ()) {
2019-10-29 09:57:25 +01:00
$errorDuplicateColumn = 'SQLSTATE[42S21]: Column already exists: 1060 Duplicate column name' ;
$errorDuplicateIndex = 'SQLSTATE[42000]: Syntax error or access violation: 1061 Duplicate key name' ;
$errorDropIndex = " /SQLSTATE \ [42000 \ ]: Syntax error or access violation: 1091 Can't DROP '[ \ w]+'; check that column \ /key exists/ " ;
$isAccepted = substr ( $errorMessage , 0 , strlen ( $errorDuplicateColumn )) === $errorDuplicateColumn ||
substr ( $errorMessage , 0 , strlen ( $errorDuplicateIndex )) === $errorDuplicateIndex ||
preg_match ( $errorDropIndex , $errorMessage ) !== 0 ;
2022-05-14 10:17:06 +02:00
} else {
2019-10-29 09:57:25 +01:00
$errorDuplicateColumn = '/ERROR: column "[\w]+" specified more than once/' ;
$errorDuplicateIndex = '/ERROR: relation "[\w]+" already exists/' ;
$errorDropIndex = '/ERROR: index "[\w]+" does not exist/' ;
$isAccepted = preg_match ( $errorDuplicateColumn , $errorMessage ) !== 0 ||
preg_match ( $errorDuplicateIndex , $errorMessage ) !== 0 ||
preg_match ( $errorDropIndex , $errorMessage ) !== 0 ;
2019-10-14 12:37:19 +02:00
}
2019-10-29 09:57:25 +01:00
return $isAccepted ;
2019-10-14 12:37:19 +02:00
}
2018-07-19 11:48:22 +02:00
// Generic update script
// add special cases where the upgrade does more than just update the DB
// this could become useful in the future
public function updateMISP ( $command )
{
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = false ;
2018-07-19 11:48:22 +02:00
switch ( $command ) {
case '2.4.20' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> ShadowAttribute = ClassRegistry :: init ( 'ShadowAttribute' );
$this -> ShadowAttribute -> upgradeToProposalCorrelation ();
break ;
case '2.4.25' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$newFeeds = array (
array ( 'provider' => 'CIRCL' , 'name' => 'CIRCL OSINT Feed' , 'url' => 'https://www.circl.lu/doc/misp/feed-osint' , 'enabled' => 0 ),
);
$this -> __addNewFeeds ( $newFeeds );
break ;
case '2.4.27' :
$newFeeds = array (
2019-10-31 08:48:00 +01:00
array ( 'provider' => 'Botvrij.eu' , 'name' => 'The Botvrij.eu Data' , 'url' => 'https://www.botvrij.eu/data/feed-osint' , 'enabled' => 0 )
2018-07-19 11:48:22 +02:00
);
$this -> __addNewFeeds ( $newFeeds );
break ;
case '2.4.49' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> SharingGroup = ClassRegistry :: init ( 'SharingGroup' );
$this -> SharingGroup -> correctSyncedSharingGroups ();
$this -> SharingGroup -> updateRoaming ();
break ;
case '2.4.55' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( 'addSightings' );
2018-07-19 11:48:22 +02:00
break ;
case '2.4.66' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( '2.4.66' );
2018-07-19 11:48:22 +02:00
$this -> cleanCacheFiles ();
$this -> Sighting = Classregistry :: init ( 'Sighting' );
$this -> Sighting -> addUuids ();
break ;
case '2.4.67' :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( '2.4.67' );
2018-07-19 11:48:22 +02:00
$this -> Sighting = Classregistry :: init ( 'Sighting' );
$this -> Sighting -> addUuids ();
$this -> Sighting -> deleteAll ( array ( 'NOT' => array ( 'Sighting.type' => array ( 0 , 1 , 2 ))));
break ;
case '2.4.71' :
2020-09-01 12:56:35 +02:00
$this -> OrgBlocklist = Classregistry :: init ( 'OrgBlocklist' );
2018-07-19 11:48:22 +02:00
$values = array (
array ( 'org_uuid' => '58d38339-7b24-4386-b4b4-4c0f950d210f' , 'org_name' => 'Setec Astrononomy' , 'comment' => 'default example' ),
array ( 'org_uuid' => '58d38326-eda8-443a-9fa8-4e12950d210f' , 'org_name' => 'Acme Finance' , 'comment' => 'default example' )
);
foreach ( $values as $value ) {
2020-09-01 12:56:35 +02:00
$found = $this -> OrgBlocklist -> find ( 'first' , array ( 'conditions' => array ( 'org_uuid' => $value [ 'org_uuid' ]), 'recursive' => - 1 ));
2018-07-19 11:48:22 +02:00
if ( empty ( $found )) {
2020-09-01 12:56:35 +02:00
$this -> OrgBlocklist -> create ();
$this -> OrgBlocklist -> save ( $value );
2018-07-19 11:48:22 +02:00
}
}
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
break ;
case '2.4.86' :
$this -> MispObject = Classregistry :: init ( 'MispObject' );
$this -> MispObject -> removeOrphanedObjects ();
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
break ;
case 5 :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> Feed = Classregistry :: init ( 'Feed' );
$this -> Feed -> setEnableFeedCachingDefaults ();
break ;
case 8 :
$this -> Server = Classregistry :: init ( 'Server' );
$this -> Server -> restartWorkers ();
break ;
case 10 :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
$this -> Role = Classregistry :: init ( 'Role' );
$this -> Role -> setPublishZmq ();
break ;
case 12 :
$this -> __forceSettings ();
break ;
2018-11-23 14:11:33 +01:00
case 23 :
$this -> __bumpReferences ();
break ;
2019-03-25 17:35:02 +01:00
case 34 :
$this -> __fixServerPullPushRules ();
break ;
2019-09-13 11:51:05 +02:00
case 38 :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2019-09-13 11:49:12 +02:00
$this -> __addServerPriority ();
break ;
2019-12-06 15:40:51 +01:00
case 46 :
2019-12-09 09:53:18 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( 'seenOnAttributeAndObject' );
2019-06-13 09:38:39 +02:00
break ;
2020-02-26 14:37:09 +01:00
case 48 :
$dbUpdateSuccess = $this -> __generateCorrelations ();
break ;
2018-07-19 11:48:22 +02:00
default :
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateDatabase ( $command );
2018-07-19 11:48:22 +02:00
break ;
}
2019-10-29 09:57:25 +01:00
return $dbUpdateSuccess ;
2018-07-19 11:48:22 +02:00
}
2019-09-13 11:49:12 +02:00
private function __addServerPriority ()
{
$this -> Server = ClassRegistry :: init ( 'Server' );
$this -> Server -> reprioritise ();
return true ;
}
2018-07-19 11:48:22 +02:00
private function __addNewFeeds ( $feeds )
{
$this -> Feed = ClassRegistry :: init ( 'Feed' );
$this -> Log = ClassRegistry :: init ( 'Log' );
$feedNames = array ();
foreach ( $feeds as $feed ) {
$feedNames [] = $feed [ 'name' ];
}
$feedNames = implode ( ', ' , $feedNames );
$result = $this -> Feed -> addDefaultFeeds ( $feeds );
$this -> Log -> create ();
$entry = array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Added new default feeds.'
2018-07-19 11:48:22 +02:00
);
if ( $result ) {
$entry [ 'change' ] = 'Feeds added: ' . $feedNames ;
} else {
$entry [ 'change' ] = 'Tried adding new feeds but something went wrong.' ;
}
$this -> Log -> save ( $entry );
}
// SQL scripts for updates
2019-05-08 15:07:26 +02:00
public function updateDatabase ( $command )
2018-07-19 11:48:22 +02:00
{
2019-08-14 15:22:29 +02:00
$this -> Log = ClassRegistry :: init ( 'Log' );
2020-03-02 23:09:47 +01:00
2019-04-29 11:09:04 +02:00
$liveOff = false ;
$exitOnError = false ;
2019-08-14 16:45:58 +02:00
if ( isset ( $this -> advanced_updates_description [ $command ])) {
$liveOff = isset ( $this -> advanced_updates_description [ $command ][ 'liveOff' ]) ? $this -> advanced_updates_description [ $command ][ 'liveOff' ] : $liveOff ;
$exitOnError = isset ( $this -> advanced_updates_description [ $command ][ 'exitOnError' ]) ? $this -> advanced_updates_description [ $command ][ 'exitOnError' ] : $exitOnError ;
2019-04-29 11:09:04 +02:00
}
2018-07-19 11:48:22 +02:00
$sqlArray = array ();
$indexArray = array ();
$clean = true ;
switch ( $command ) {
case 'extendServerOrganizationLength' :
$sqlArray [] = 'ALTER TABLE `servers` MODIFY COLUMN `organization` varchar(255) NOT NULL;' ;
break ;
case 'convertLogFieldsToText' :
$sqlArray [] = 'ALTER TABLE `logs` MODIFY COLUMN `title` text, MODIFY COLUMN `change` text;' ;
break ;
case 'addEventBlacklists' :
$sqlArray [] = 'CREATE TABLE IF NOT EXISTS `event_blacklists` ( `id` int(11) NOT NULL AUTO_INCREMENT, `event_uuid` varchar(40) COLLATE utf8_bin NOT NULL, `created` datetime NOT NULL, PRIMARY KEY (`id`), `event_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, `event_orgc` VARCHAR( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;' ;
break ;
case 'addOrgBlacklists' :
$sqlArray [] = 'CREATE TABLE IF NOT EXISTS `org_blacklists` ( `id` int(11) NOT NULL AUTO_INCREMENT, `org_uuid` varchar(40) COLLATE utf8_bin NOT NULL, `created` datetime NOT NULL, PRIMARY KEY (`id`), `org_name` varchar(255) COLLATE utf8_bin NOT NULL, `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;' ;
break ;
case 'addEventBlacklistsContext' :
$sqlArray [] = 'ALTER TABLE `event_blacklists` ADD `event_orgc` VARCHAR( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL , ADD `event_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, ADD `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL;' ;
break ;
case 'addSightings' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS sightings (
2019-02-10 13:08:12 +01:00
id int ( 11 ) NOT NULL AUTO_INCREMENT ,
attribute_id int ( 11 ) NOT NULL ,
event_id int ( 11 ) NOT NULL ,
org_id int ( 11 ) NOT NULL ,
date_sighting bigint ( 20 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX attribute_id ( attribute_id ),
INDEX event_id ( event_id ),
INDEX org_id ( org_id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 'makeAttributeUUIDsUnique' :
$this -> __dropIndex ( 'attributes' , 'uuid' );
$sqlArray [] = 'ALTER TABLE `attributes` ADD UNIQUE (uuid);' ;
break ;
case 'makeEventUUIDsUnique' :
$this -> __dropIndex ( 'events' , 'uuid' );
$sqlArray [] = 'ALTER TABLE `events` ADD UNIQUE (uuid);' ;
break ;
case 'cleanSessionTable' :
$sqlArray [] = 'DELETE FROM cake_sessions WHERE expires < ' . time () . ';' ;
$clean = false ;
break ;
case 'destroyAllSessions' :
$sqlArray [] = 'DELETE FROM cake_sessions;' ;
$clean = false ;
break ;
case 'addIPLogging' :
$sqlArray [] = 'ALTER TABLE `logs` ADD `ip` varchar(45) COLLATE utf8_bin DEFAULT NULL;' ;
break ;
case 'addCustomAuth' :
$sqlArray [] = " ALTER TABLE `users` ADD `external_auth_required` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `users` ADD `external_auth_key` text COLLATE utf8_bin;' ;
break ;
2018-10-22 10:36:13 +02:00
case 'x24betaupdates' :
2018-07-19 11:48:22 +02:00
$sqlArray = array ();
$sqlArray [] = " ALTER TABLE `shadow_attributes` ADD `proposal_to_delete` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `logs` MODIFY `change` text COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `taxonomies` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`namespace` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`description` text COLLATE utf8_bin NOT NULL ,
`version` int ( 11 ) NOT NULL ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `taxonomy_entries` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`taxonomy_predicate_id` int ( 11 ) NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`expanded` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` ),
KEY `taxonomy_predicate_id` ( `taxonomy_predicate_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `taxonomy_predicates` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`taxonomy_id` int ( 11 ) NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`expanded` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` ),
KEY `taxonomy_id` ( `taxonomy_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-06-04 01:08:16 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'ALTER TABLE `jobs` ADD `org` text COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `servers` ADD `name` varchar(255) NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `sharing_groups` ADD `sync_user_id` INT( 11 ) NOT NULL DEFAULT \'0\' AFTER `org_id`;' ;
$sqlArray [] = 'ALTER TABLE `users` ADD `disabled` BOOLEAN NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `users` ADD `expiration` datetime DEFAULT NULL;' ;
$sqlArray [] = 'UPDATE `roles` SET `perm_template` = 1 WHERE `perm_site_admin` = 1 OR `perm_admin` = 1;' ;
$sqlArray [] = 'UPDATE `roles` SET `perm_sharing_group` = 1 WHERE `perm_site_admin` = 1 OR `perm_sync` = 1;' ;
//create indexes
break ;
case 'indexTables' :
$fieldsToIndex = array (
'attributes' => array ( array ( 'value1' , 'INDEX' , '255' ), array ( 'value2' , 'INDEX' , '255' ), array ( 'event_id' , 'INDEX' ), array ( 'sharing_group_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' )),
'correlations' => array ( array ( 'org_id' , 'INDEX' ), array ( 'event_id' , 'INDEX' ), array ( 'attribute_id' , 'INDEX' ), array ( 'sharing_group_id' , 'INDEX' ), array ( '1_event_id' , 'INDEX' ), array ( '1_attribute_id' , 'INDEX' ), array ( 'a_sharing_group_id' , 'INDEX' ), array ( 'value' , 'FULLTEXT' )),
'events' => array ( array ( 'info' , 'FULLTEXT' ), array ( 'sharing_group_id' , 'INDEX' ), array ( 'org_id' , 'INDEX' ), array ( 'orgc_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' )),
'event_tags' => array ( array ( 'event_id' , 'INDEX' ), array ( 'tag_id' , 'INDEX' )),
'organisations' => array ( array ( 'uuid' , 'INDEX' ), array ( 'name' , 'FULLTEXT' )),
'posts' => array ( array ( 'post_id' , 'INDEX' ), array ( 'thread_id' , 'INDEX' )),
'shadow_attributes' => array ( array ( 'value1' , 'INDEX' , '255' ), array ( 'value2' , 'INDEX' , '255' ), array ( 'old_id' , 'INDEX' ), array ( 'event_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' ), array ( 'event_org_id' , 'INDEX' ), array ( 'event_uuid' , 'INDEX' )),
'sharing_groups' => array ( array ( 'org_id' , 'INDEX' ), array ( 'sync_user_id' , 'INDEX' ), array ( 'uuid' , 'INDEX' ), array ( 'organisation_uuid' , 'INDEX' )),
'sharing_group_orgs' => array ( array ( 'sharing_group_id' , 'INDEX' ), array ( 'org_id' , 'INDEX' )),
'sharing_group_servers' => array ( array ( 'sharing_group_id' , 'INDEX' ), array ( 'server_id' , 'INDEX' )),
'servers' => array ( array ( 'org_id' , 'INDEX' ), array ( 'remote_org_id' , 'INDEX' )),
'tags' => array ( array ( 'name' , 'FULLTEXT' )),
'threads' => array ( array ( 'user_id' , 'INDEX' ), array ( 'event_id' , 'INDEX' ), array ( 'org_id' , 'INDEX' ), array ( 'sharing_group_id' , 'INDEX' )),
'users' => array ( array ( 'org_id' , 'INDEX' ), array ( 'server_id' , 'INDEX' ), array ( 'email' , 'INDEX' )),
);
$version = $this -> query ( 'select version();' );
$version = $version [ 0 ][ 0 ][ 'version()' ];
$version = explode ( '.' , $version );
$version [ 0 ] = intval ( $version [ 0 ]);
$version [ 1 ] = intval ( $version [ 1 ]);
$downgrade = true ;
if ( $version [ 0 ] > 5 || ( $version [ 0 ] == 5 && $version [ 1 ] > 5 )) {
$downgrade = false ;
}
// keep the fulltext for now, we can change it later to actually use it once we require MySQL 5.6 / or if we decide to move some tables to MyISAM
foreach ( $fieldsToIndex as $table => $fields ) {
$downgradeThis = false ;
$table_data = $this -> query ( " SHOW TABLE STATUS WHERE Name = ' " . $table . " ' " );
if ( $downgrade && $table_data [ 0 ][ 'TABLES' ][ 'Engine' ] !== 'MyISAM' ) {
$downgradeThis = true ;
}
foreach ( $fields as $field ) {
$extra = '' ;
$this -> __dropIndex ( $table , $field [ 0 ]);
if ( isset ( $field [ 2 ])) {
$extra = ' (' . $field [ 2 ] . ')' ;
}
$sqlArray [] = 'ALTER TABLE `' . $table . '` ADD ' . ( $downgradeThis ? 'INDEX' : $field [ 1 ]) . ' `' . $field [ 0 ] . '` (`' . $field [ 0 ] . '`' . $extra . ');' ;
}
}
break ;
case 'adminTable' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `admin_settings` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`setting` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = " INSERT INTO `admin_settings` (`setting`, `value`) VALUES ('db_version', '2.4.0'); " ;
break ;
case '2.4.18' :
$sqlArray [] = " ALTER TABLE `users` ADD `current_login` INT(11) DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ADD `last_login` INT(11) DEFAULT 0; " ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `event_delegations` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`requester_org_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`message` text ,
`distribution` tinyint ( 4 ) NOT NULL DEFAULT '-1' ,
`sharing_group_id` int ( 11 ),
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `event_id` ( `event_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.19' :
$sqlArray [] = " DELETE FROM `shadow_attributes` WHERE `event_uuid` = ''; " ;
break ;
case '2.4.20' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `shadow_attribute_correlations` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`value` text NOT NULL ,
`distribution` tinyint ( 4 ) NOT NULL ,
`a_distribution` tinyint ( 4 ) NOT NULL ,
`sharing_group_id` int ( 11 ),
`a_sharing_group_id` int ( 11 ),
`attribute_id` int ( 11 ) NOT NULL ,
`1_shadow_attribute_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`1_event_id` int ( 11 ) NOT NULL ,
`info` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `attribute_id` ( `attribute_id` ),
KEY `a_sharing_group_id` ( `a_sharing_group_id` ),
KEY `event_id` ( `event_id` ),
KEY `1_event_id` ( `event_id` ),
KEY `sharing_group_id` ( `sharing_group_id` ),
KEY `1_shadow_attribute_id` ( `1_shadow_attribute_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.25' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `feeds` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`provider` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`url` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`rules` text COLLATE utf8_bin NOT NULL ,
`enabled` BOOLEAN NOT NULL ,
`distribution` tinyint ( 4 ) NOT NULL ,
`sharing_group_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
`default` tinyint ( 1 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.32' :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_tag_editor` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'UPDATE `roles` SET `perm_tag_editor` = 1 WHERE `perm_tagger` = 1;' ;
break ;
case '2.4.33' :
$sqlArray [] = " ALTER TABLE `users` ADD `force_logout` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case '2.4.38' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `warninglists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT 'string' ,
`description` text COLLATE utf8_bin NOT NULL ,
`version` int ( 11 ) NOT NULL DEFAULT 1 ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`warninglist_entry_count` int ( 11 ) unsigned DEFAULT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `warninglist_entries` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`value` text CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`warninglist_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `warninglist_types` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`warninglist_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.39' :
$sqlArray [] = " ALTER TABLE `users` ADD `certif_public` longtext COLLATE utf8_bin AFTER `gpgkey`; " ;
$sqlArray [] = 'ALTER TABLE `logs` MODIFY COLUMN `title` text, MODIFY COLUMN `change` text;' ;
break ;
case '2.4.40' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `favourite_tags` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`tag_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` ),
INDEX `user_id` ( `user_id` ),
INDEX `tag_id` ( `tag_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.42' :
$sqlArray [] = " ALTER TABLE `attributes` ADD `deleted` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case '2.4.44' :
$sqlArray [] = " UPDATE `servers` SET `url` = TRIM(TRAILING '/' FROM `url`); " ;
break ;
case '2.4.45' :
$sqlArray [] = 'ALTER TABLE `users` CHANGE `newsread` `newsread` int(11) unsigned;' ;
$sqlArray [] = 'UPDATE `users` SET `newsread` = 0;' ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `news` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`message` text COLLATE utf8_bin NOT NULL ,
`title` text COLLATE utf8_bin NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`date_created` int ( 11 ) unsigned NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case '2.4.49' :
// table: users
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `server_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `autoalert` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `invited_by` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `nids_sid` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `termsaccepted` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `role_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `change_pw` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `contactalert` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` ALTER COLUMN `disabled` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `users` MODIFY `authkey` varchar(40) COLLATE utf8_bin DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `users` MODIFY `gpgkey` longtext COLLATE utf8_bin; " ;
// table: events
$sqlArray [] = " ALTER TABLE `events` ALTER COLUMN `publish_timestamp` SET DEFAULT 0; " ;
// table: jobs
$sqlArray [] = " ALTER TABLE `jobs` ALTER COLUMN `org_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `jobs` MODIFY `process_id` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
// table: organisations
$sqlArray [] = " ALTER TABLE `organisations` ALTER COLUMN `created_by` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `organisations` MODIFY `uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL; " ; // https://github.com/MISP/MISP/pull/1260
// table: logs
$sqlArray [] = " ALTER TABLE `logs` MODIFY `title` text CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `logs` MODIFY `change` text CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `logs` MODIFY `description` text CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
// table: servers
$sqlArray [] = " ALTER TABLE `servers` DROP `lastfetchedid`; " ; // git commit hash d4c393897e8666fbbf04443a97d60c508700f5b4
$sqlArray [] = " ALTER TABLE `servers` MODIFY `cert_file` varchar(255) COLLATE utf8_bin DEFAULT NULL; " ;
// table: feeds
$sqlArray [] = " ALTER TABLE `feeds` ALTER COLUMN `sharing_group_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `feeds` ALTER COLUMN `tag_id` SET DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `feeds` MODIFY `rules` text COLLATE utf8_bin DEFAULT NULL; " ;
// DB changes to support https://github.com/MISP/MISP/pull/1334
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_delegate` tinyint(1) NOT NULL DEFAULT 0 AFTER `perm_publish`; " ;
$sqlArray [] = " UPDATE `roles` SET `perm_delegate` = 1 WHERE `perm_publish` = 1; " ;
// DB changes to solve https://github.com/MISP/MISP/issues/1354
$sqlArray [] = " ALTER TABLE `taxonomy_entries` MODIFY `expanded` text COLLATE utf8_bin; " ;
$sqlArray [] = " ALTER TABLE `taxonomy_predicates` MODIFY `expanded` text COLLATE utf8_bin; " ;
// Sharing group propagate to instances freely setting
$sqlArray [] = " ALTER TABLE `sharing_groups` ADD `roaming` tinyint(1) NOT NULL DEFAULT 0; " ;
// table: shadow_attributes
$sqlArray [] = " ALTER TABLE `shadow_attributes` MODIFY `email` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL; " ;
// table: tasks
$sqlArray [] = " ALTER TABLE `tasks` CHANGE `job_id` `process_id` varchar(32) DEFAULT NULL; " ;
// Adding tag org restrictions
$sqlArray [] = " ALTER TABLE `tags` ADD `org_id` int(11) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `tags` ADD INDEX `org_id` (`org_id`);' ;
$this -> __dropIndex ( 'tags' , 'org_id' );
break ;
case '2.4.50' :
$sqlArray [] = 'ALTER TABLE `cake_sessions` ADD INDEX `expires` (`expires`);' ;
$sqlArray [] = " ALTER TABLE `users` ADD `certif_public` longtext COLLATE utf8_bin AFTER `gpgkey`; " ;
$sqlArray [] = " ALTER TABLE `servers` ADD `client_cert_file` varchar(255) COLLATE utf8_bin DEFAULT NULL; " ;
$this -> __dropIndex ( 'cake_sessions' , 'expires' );
break ;
case '2.4.51' :
$sqlArray [] = 'ALTER TABLE `servers` ADD `internal` tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE `roles` ADD `default_role` tinyint(1) NOT NULL DEFAULT 0;' ;
break ;
case '2.4.52' :
$sqlArray [] = " ALTER TABLE feeds ADD source_format varchar(255) COLLATE utf8_bin DEFAULT 'misp'; " ;
$sqlArray [] = 'ALTER TABLE feeds ADD fixed_event tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD delta_merge tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD event_id int(11) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD publish tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds ADD override_ids tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = " ALTER TABLE feeds ADD settings text NOT NULL DEFAULT ''; " ;
break ;
case '2.4.56' :
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxies (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`description` text COLLATE utf8_bin NOT NULL ,
`version` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxies' , 'name' );
$this -> __addIndex ( 'galaxies' , 'uuid' );
$this -> __addIndex ( 'galaxies' , 'type' );
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxy_clusters (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`tag_name` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`description` text COLLATE utf8_bin NOT NULL ,
`galaxy_id` int ( 11 ) NOT NULL ,
`source` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`authors` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxy_clusters' , 'value' , 255 );
$this -> __addIndex ( 'galaxy_clusters' , 'tag_name' );
$this -> __addIndex ( 'galaxy_clusters' , 'uuid' );
$this -> __addIndex ( 'galaxy_clusters' , 'type' );
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxy_elements (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`galaxy_cluster_id` int ( 11 ) NOT NULL ,
`key` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
`value` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxy_elements' , 'key' );
$this -> __addIndex ( 'galaxy_elements' , 'value' , 255 );
2016-12-04 20:25:47 +01:00
2018-07-19 11:48:22 +02:00
$sqlArray [] =
" CREATE TABLE IF NOT EXISTS galaxy_reference (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`galaxy_cluster_id` int ( 11 ) NOT NULL ,
`referenced_galaxy_cluster_id` int ( 11 ) NOT NULL ,
`referenced_galaxy_cluster_uuid` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`referenced_galaxy_cluster_type` text COLLATE utf8_bin NOT NULL ,
`referenced_galaxy_cluster_value` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2016-12-16 15:44:10 +01:00
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'galaxy_reference' , 'galaxy_cluster_id' );
$this -> __addIndex ( 'galaxy_reference' , 'referenced_galaxy_cluster_id' );
$this -> __addIndex ( 'galaxy_reference' , 'referenced_galaxy_cluster_value' , 255 );
$this -> __addIndex ( 'galaxy_reference' , 'referenced_galaxy_cluster_type' , 255 );
break ;
case '2.4.57' :
$sqlArray [] = 'ALTER TABLE tags ADD hide_tag tinyint(1) NOT NULL DEFAULT 0;' ;
// new indeces to match the changes in #1766
$this -> __dropIndex ( 'correlations' , '1_event_id' );
$this -> __addIndex ( 'correlations' , '1_event_id' );
$this -> __addIndex ( 'warninglist_entries' , 'warninglist_id' );
break ;
case '2.4.58' :
$sqlArray [] = " ALTER TABLE `events` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `attributes` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case '2.4.59' :
$sqlArray [] = " ALTER TABLE taxonomy_entries ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT ''; " ;
$sqlArray [] = " ALTER TABLE taxonomy_predicates ADD colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT ''; " ;
break ;
case '2.4.60' :
2022-05-14 10:17:06 +02:00
if ( $this -> isMysql ()) {
2018-07-19 11:48:22 +02:00
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `attribute_tags` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`attribute_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; ' ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'ALTER TABLE `attribute_tags` ADD INDEX `attribute_id` (`attribute_id`);' ;
$sqlArray [] = 'ALTER TABLE `attribute_tags` ADD INDEX `event_id` (`event_id`);' ;
$sqlArray [] = 'ALTER TABLE `attribute_tags` ADD INDEX `tag_id` (`tag_id`);' ;
2022-05-14 10:17:06 +02:00
} else {
2018-07-19 11:48:22 +02:00
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS attribute_tags (
2019-02-10 13:08:12 +01:00
id bigserial NOT NULL ,
attribute_id bigint NOT NULL ,
event_id bigint NOT NULL ,
tag_id bigint NOT NULL ,
PRIMARY KEY ( id )
); ' ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'CREATE INDEX idx_attribute_tags_attribute_id ON attribute_tags (attribute_id);' ;
$sqlArray [] = 'CREATE INDEX idx_attribute_tags_event_id ON attribute_tags (event_id);' ;
$sqlArray [] = 'CREATE INDEX idx_attribute_tags_tag_id ON attribute_tags (tag_id);' ;
}
break ;
case '2.4.61' :
$sqlArray [] = 'ALTER TABLE feeds ADD input_source varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "network";' ;
$sqlArray [] = 'ALTER TABLE feeds ADD delete_local_file tinyint(1) DEFAULT 0;' ;
$indexArray [] = array ( 'feeds' , 'input_source' );
break ;
case '2.4.62' :
$sqlArray [] = 'ALTER TABLE logs CHANGE `org` `org` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE logs CHANGE `email` `email` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE logs CHANGE `change` `change` text COLLATE utf8_bin NOT NULL DEFAULT "";' ;
break ;
case '2.4.63' :
$sqlArray [] = 'ALTER TABLE events DROP COLUMN org;' ;
$sqlArray [] = 'ALTER TABLE events DROP COLUMN orgc;' ;
$sqlArray [] = 'ALTER TABLE event_blacklists CHANGE comment comment TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci;' ;
break ;
case '2.4.64' :
$indexArray [] = array ( 'feeds' , 'input_source' );
$indexArray [] = array ( 'attributes' , 'value1' , 255 );
$indexArray [] = array ( 'attributes' , 'value2' , 255 );
$indexArray [] = array ( 'attributes' , 'type' );
$indexArray [] = array ( 'galaxy_reference' , 'galaxy_cluster_id' );
$indexArray [] = array ( 'galaxy_reference' , 'referenced_galaxy_cluster_id' );
$indexArray [] = array ( 'galaxy_reference' , 'referenced_galaxy_cluster_value' , 255 );
$indexArray [] = array ( 'galaxy_reference' , 'referenced_galaxy_cluster_type' , 255 );
$indexArray [] = array ( 'correlations' , '1_event_id' );
$indexArray [] = array ( 'warninglist_entries' , 'warninglist_id' );
$indexArray [] = array ( 'galaxy_clusters' , 'value' , 255 );
$indexArray [] = array ( 'galaxy_clusters' , 'tag_name' );
$indexArray [] = array ( 'galaxy_clusters' , 'uuid' );
$indexArray [] = array ( 'galaxy_clusters' , 'type' );
$indexArray [] = array ( 'galaxies' , 'name' );
$indexArray [] = array ( 'galaxies' , 'uuid' );
$indexArray [] = array ( 'galaxies' , 'type' );
break ;
case '2.4.65' :
$sqlArray [] = 'ALTER TABLE feeds CHANGE `enabled` `enabled` tinyint(1) DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds CHANGE `default` `default` tinyint(1) DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds CHANGE `distribution` `distribution` tinyint(4) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE feeds CHANGE `sharing_group_id` `sharing_group_id` int(11) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE attributes CHANGE `comment` `comment` text COLLATE utf8_bin;' ;
break ;
case '2.4.66' :
$sqlArray [] = 'ALTER TABLE shadow_attributes CHANGE old_id old_id int(11) DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE sightings ADD COLUMN uuid varchar(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE sightings ADD COLUMN source varchar(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE sightings ADD COLUMN type int(11) DEFAULT 0;' ;
$indexArray [] = array ( 'sightings' , 'uuid' );
$indexArray [] = array ( 'sightings' , 'source' );
$indexArray [] = array ( 'sightings' , 'type' );
$indexArray [] = array ( 'attributes' , 'category' );
$indexArray [] = array ( 'shadow_attributes' , 'category' );
$indexArray [] = array ( 'shadow_attributes' , 'type' );
break ;
case '2.4.67' :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_sighting` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'UPDATE `roles` SET `perm_sighting` = 1 WHERE `perm_add` = 1;' ;
break ;
case '2.4.68' :
$sqlArray [] = 'ALTER TABLE events CHANGE attribute_count attribute_count int(11) unsigned DEFAULT 0;' ;
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `event_blacklists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`created` datetime NOT NULL ,
`event_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`event_orgc` VARCHAR ( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; ' ;
2018-07-19 11:48:22 +02:00
$indexArray [] = array ( 'event_blacklists' , 'event_uuid' );
$indexArray [] = array ( 'event_blacklists' , 'event_orgc' );
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `org_blacklists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`created` datetime NOT NULL ,
`org_name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
PRIMARY KEY ( `id` ),
INDEX `org_uuid` ( `org_uuid` ),
INDEX `org_name` ( `org_name` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; ' ;
2018-07-19 11:48:22 +02:00
$indexArray [] = array ( 'org_blacklists' , 'org_uuid' );
$indexArray [] = array ( 'org_blacklists' , 'org_name' );
$sqlArray [] = " ALTER TABLE shadow_attributes CHANGE proposal_to_delete proposal_to_delete BOOLEAN DEFAULT 0 " ;
$sqlArray [] = " ALTER TABLE taxonomy_predicates CHANGE colour colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin; " ;
$sqlArray [] = " ALTER TABLE taxonomy_entries CHANGE colour colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin; " ;
break ;
case '2.4.69' :
$sqlArray [] = " ALTER TABLE taxonomy_entries CHANGE colour colour varchar(7) CHARACTER SET utf8 COLLATE utf8_bin; " ;
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_created bigint(20); " ;
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_modified bigint(20); " ;
break ;
case '2.4.71' :
$sqlArray [] = " UPDATE attributes SET comment = '' WHERE comment is NULL; " ;
$sqlArray [] = " ALTER TABLE attributes CHANGE comment comment text COLLATE utf8_bin NOT NULL; " ;
break ;
case '2.4.72' :
$sqlArray [] = 'ALTER TABLE feeds ADD lookup_visible tinyint(1) DEFAULT 0;' ;
break ;
case '2.4.73' :
$sqlArray [] = 'ALTER TABLE `servers` ADD `unpublish_event` tinyint(1) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE `servers` ADD `publish_without_email` tinyint(1) NOT NULL DEFAULT 0;' ;
break ;
case '2.4.75' :
$this -> __dropIndex ( 'attributes' , 'value1' );
$this -> __dropIndex ( 'attributes' , 'value2' );
$this -> __addIndex ( 'attributes' , 'value1' , 255 );
$this -> __addIndex ( 'attributes' , 'value2' , 255 );
break ;
case '2.4.77' :
$sqlArray [] = 'ALTER TABLE `users` CHANGE `password` `password` VARCHAR(255) COLLATE utf8_bin NOT NULL;' ;
break ;
case '2.4.78' :
$sqlArray [] = " ALTER TABLE galaxy_clusters ADD COLUMN version int(11) DEFAULT 0; " ;
$this -> __addIndex ( 'galaxy_clusters' , 'version' );
$this -> __addIndex ( 'galaxy_clusters' , 'galaxy_id' );
$this -> __addIndex ( 'galaxy_elements' , 'galaxy_cluster_id' );
break ;
case '2.4.80' :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS objects (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`meta-category` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`description` text CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`template_uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`template_version` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`distribution` tinyint ( 4 ) NOT NULL DEFAULT 0 ,
`sharing_group_id` int ( 11 ),
`comment` text COLLATE utf8_bin NOT NULL ,
`deleted` TINYINT ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` ),
INDEX `template_uuid` ( `template_uuid` ),
INDEX `template_version` ( `template_version` ),
INDEX `meta-category` ( `meta-category` ),
INDEX `event_id` ( `event_id` ),
INDEX `uuid` ( `uuid` ),
INDEX `timestamp` ( `timestamp` ),
INDEX `distribution` ( `distribution` ),
INDEX `sharing_group_id` ( `sharing_group_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_references (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`object_id` int ( 11 ) NOT NULL ,
`event_id` int ( 11 ) NOT NULL ,
`object_uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`referenced_uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`referenced_id` int ( 11 ) NOT NULL ,
`referenced_type` int ( 11 ) NOT NULL DEFAULT 0 ,
`relationship_type` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`comment` text COLLATE utf8_bin NOT NULL ,
`deleted` TINYINT ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `object_uuid` ( `object_uuid` ),
INDEX `referenced_uuid` ( `referenced_uuid` ),
INDEX `timestamp` ( `timestamp` ),
INDEX `object_id` ( `object_id` ),
INDEX `referenced_id` ( `referenced_id` ),
INDEX `relationship_type` ( `relationship_type` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_relationships (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`version` int ( 11 ) NOT NULL ,
`name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`description` text COLLATE utf8_bin NOT NULL ,
`format` text COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-08-21 10:13:19 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_templates (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`meta-category` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`description` text COLLATE utf8_bin ,
`version` int ( 11 ) NOT NULL ,
`requirements` text COLLATE utf8_bin ,
`fixed` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`active` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` ),
INDEX `uuid` ( `uuid` ),
INDEX `name` ( `name` ),
INDEX `meta-category` ( `meta-category` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS object_template_elements (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`object_template_id` int ( 11 ) NOT NULL ,
`object_relation` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`type` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`ui-priority` int ( 11 ) NOT NULL ,
`categories` text COLLATE utf8_bin ,
`sane_default` text COLLATE utf8_bin ,
`values_list` text COLLATE utf8_bin ,
`description` text COLLATE utf8_bin ,
`disable_correlation` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`multiple` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `object_relation` ( `object_relation` ),
INDEX `type` ( `type` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2017-06-13 12:01:03 +02:00
2018-07-19 11:48:22 +02:00
$sqlArray [] = 'ALTER TABLE `logs` CHANGE `model` `model` VARCHAR(80) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `logs` CHANGE `action` `action` VARCHAR(80) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE attributes ADD object_id int(11) NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE attributes ADD object_relation varchar(255) COLLATE utf8_bin;' ;
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_object_template` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'UPDATE `roles` SET `perm_object_template` = 1 WHERE `perm_site_admin` = 1;' ;
$indexArray [] = array ( 'attributes' , 'object_id' );
$indexArray [] = array ( 'attributes' , 'object_relation' );
break ;
case '2.4.81' :
$sqlArray [] = 'ALTER TABLE `galaxy_clusters` ADD `version` INT NOT NULL DEFAULT 0;' ;
$sqlArray [] = 'ALTER TABLE `galaxies` ADD `icon` VARCHAR(255) COLLATE utf8_bin DEFAULT "";' ;
break ;
case '2.4.82' :
$sqlArray [] = " ALTER TABLE organisations ADD restricted_to_domain text COLLATE utf8_bin; " ;
break ;
case '2.4.83' :
$sqlArray [] = " ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` text COLLATE utf8_bin; " ;
break ;
case '2.4.84' :
$sqlArray [] = " ALTER TABLE `tags` ADD `user_id` int(11) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `tags` ADD INDEX `user_id` (`user_id`);' ;
break ;
case '2.4.85' :
$sqlArray [] = " ALTER TABLE `shadow_attributes` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` text COLLATE utf8_bin; " ;
// yes, this may look stupid as hell to index a boolean flag - but thanks to the stupidity of MySQL/MariaDB this will
// stop blocking other indexes to be used in queries where we also tests for the deleted flag.
$indexArray [] = array ( 'attributes' , 'deleted' );
break ;
case '2.4.86' :
break ;
case '2.4.87' :
$sqlArray [] = " ALTER TABLE `feeds` ADD `headers` TEXT COLLATE utf8_bin; " ;
break ;
case 1 :
$sqlArray [] = " ALTER TABLE `tags` ADD `user_id` int(11) NOT NULL DEFAULT 0; " ;
$sqlArray [] = 'ALTER TABLE `tags` ADD INDEX `user_id` (`user_id`);' ;
break ;
case 2 :
// rerun missing db entries
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_created bigint(20); " ;
$sqlArray [] = " ALTER TABLE users ADD COLUMN date_modified bigint(20); " ;
break ;
case 3 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `fuzzy_correlate_ssdeep` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`chunk` varchar ( 12 ) NOT NULL ,
`attribute_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
$this -> __addIndex ( 'fuzzy_correlate_ssdeep' , 'chunk' );
$this -> __addIndex ( 'fuzzy_correlate_ssdeep' , 'attribute_id' );
break ;
case 4 :
$sqlArray [] = 'ALTER TABLE `roles` ADD `memory_limit` VARCHAR(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = 'ALTER TABLE `roles` ADD `max_execution_time` VARCHAR(255) COLLATE utf8_bin DEFAULT "";' ;
$sqlArray [] = " ALTER TABLE `roles` ADD `restricted_to_site_admin` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 5 :
$sqlArray [] = " ALTER TABLE `feeds` ADD `caching_enabled` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 6 :
$sqlArray [] = " ALTER TABLE `events` ADD `extends_uuid` varchar(40) COLLATE utf8_bin DEFAULT ''; " ;
$indexArray [] = array ( 'events' , 'extends_uuid' );
break ;
case 7 :
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `noticelists` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) COLLATE utf8_unicode_ci NOT NULL ,
`expanded_name` text COLLATE utf8_unicode_ci NOT NULL ,
`ref` text COLLATE utf8_unicode_ci ,
`geographical_area` varchar ( 255 ) COLLATE utf8_unicode_ci ,
`version` int ( 11 ) NOT NULL DEFAULT 1 ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( `id` ),
INDEX `name` ( `name` ),
INDEX `geographical_area` ( `geographical_area` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; ' ;
2018-07-19 11:48:22 +02:00
$sqlArray [] = ' CREATE TABLE IF NOT EXISTS `noticelist_entries` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`noticelist_id` int ( 11 ) NOT NULL ,
`data` text COLLATE utf8_unicode_ci NOT NULL ,
PRIMARY KEY ( `id` ),
INDEX `noticelist_id` ( `noticelist_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; ' ;
2018-07-19 11:48:22 +02:00
break ;
case 9 :
$sqlArray [] = 'ALTER TABLE galaxies ADD namespace varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT "misp";' ;
$indexArray [] = array ( 'galaxies' , 'namespace' );
break ;
case 10 :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_publish_zmq` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 11 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_locks (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `event_id` ( `event_id` ),
INDEX `user_id` ( `user_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 12 :
$sqlArray [] = " ALTER TABLE `servers` ADD `skip_proxy` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 13 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_graph (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`network_name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`network_json` MEDIUMTEXT NOT NULL ,
`preview_img` MEDIUMTEXT ,
PRIMARY KEY ( id ),
INDEX `event_id` ( `event_id` ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 14 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `user_settings` (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`setting` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`value` text COLLATE utf8_bin NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
INDEX `setting` ( `setting` ),
INDEX `user_id` ( `user_id` ),
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
case 15 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_graph (
2019-02-10 13:08:12 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`event_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`network_name` varchar ( 255 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`network_json` MEDIUMTEXT NOT NULL ,
`preview_img` MEDIUMTEXT ,
PRIMARY KEY ( id ),
INDEX `event_id` ( `event_id` ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-07-19 11:48:22 +02:00
break ;
2018-09-10 07:52:27 +02:00
case 18 :
2018-07-22 23:28:07 +02:00
$sqlArray [] = 'ALTER TABLE `taxonomy_predicates` ADD COLUMN description text CHARACTER SET UTF8 collate utf8_bin;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_entries` ADD COLUMN description text CHARACTER SET UTF8 collate utf8_bin;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_predicates` ADD COLUMN exclusive tinyint(1) DEFAULT 0;' ;
break ;
2018-09-10 07:52:27 +02:00
case 19 :
2018-07-26 17:13:38 +02:00
$sqlArray [] = 'ALTER TABLE `taxonomies` ADD COLUMN exclusive tinyint(1) DEFAULT 0;' ;
break ;
2018-09-10 22:02:07 +02:00
case 20 :
$sqlArray [] = " ALTER TABLE `servers` ADD `skip_proxy` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
case 21 :
$sqlArray [] = 'ALTER TABLE `tags` ADD COLUMN numerical_value int(11) NULL;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_predicates` ADD COLUMN numerical_value int(11) NULL;' ;
$sqlArray [] = 'ALTER TABLE `taxonomy_entries` ADD COLUMN numerical_value int(11) NULL;' ;
break ;
2018-11-23 14:11:33 +01:00
case 22 :
$sqlArray [] = 'ALTER TABLE `object_references` MODIFY `deleted` tinyint(1) NOT NULL default 0;' ;
break ;
case 24 :
$this -> GalaxyCluster = ClassRegistry :: init ( 'GalaxyCluster' );
if ( empty ( $this -> GalaxyCluster -> schema ( 'collection_uuid' ))) {
$sqlArray [] = 'ALTER TABLE `galaxy_clusters` CHANGE `uuid` `collection_uuid` varchar(255) COLLATE utf8_bin NOT NULL;' ;
$sqlArray [] = 'ALTER TABLE `galaxy_clusters` ADD COLUMN `uuid` varchar(255) COLLATE utf8_bin NOT NULL default \'\';' ;
}
break ;
case 25 :
$this -> __dropIndex ( 'galaxy_clusters' , 'uuid' );
$this -> __addIndex ( 'galaxy_clusters' , 'uuid' );
$this -> __addIndex ( 'galaxy_clusters' , 'collection_uuid' );
break ;
2018-12-15 10:02:38 +01:00
case 26 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS tag_collections (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`name` varchar ( 255 ) COLLATE utf8_unicode_ci NOT NULL ,
`description` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`all_orgs` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `uuid` ( `uuid` ),
INDEX `user_id` ( `user_id` ),
INDEX `org_id` ( `org_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2018-12-27 05:24:30 +01:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS tag_collection_tags (
2018-12-15 10:02:38 +01:00
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`tag_collection_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `uuid` ( `tag_collection_id` ),
INDEX `user_id` ( `tag_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
break ;
2019-01-01 16:37:15 +01:00
case 27 :
$sqlArray [] = 'ALTER TABLE `tags` CHANGE `org_id` `org_id` int(11) NOT NULL DEFAULT 0;' ;
break ;
2019-01-18 09:06:23 +01:00
case 28 :
$sqlArray [] = " ALTER TABLE `servers` ADD `caching_enabled` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-02-14 10:24:42 +01:00
case 29 :
$sqlArray [] = " ALTER TABLE `galaxies` ADD `kill_chain_order` text NOT NULL; " ;
break ;
2019-02-22 08:04:23 +01:00
case 30 :
$sqlArray [] = " ALTER TABLE `galaxies` MODIFY COLUMN `kill_chain_order` text " ;
$sqlArray [] = " ALTER TABLE `feeds` ADD `force_to_ids` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-03-19 10:55:27 +01:00
case 31 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `rest_client_histories` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`headers` text ,
`body` text ,
`url` text ,
`http_method` varchar ( 255 ),
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`use_full_path` tinyint ( 1 ) DEFAULT 0 ,
`show_result` tinyint ( 1 ) DEFAULT 0 ,
`skip_ssl` tinyint ( 1 ) DEFAULT 0 ,
`outcome` int ( 11 ) NOT NULL ,
`bookmark` tinyint ( 1 ) NOT NULL DEFAUlT 0 ,
`bookmark_name` varchar ( 255 ) NULL DEFAULT '' ,
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `user_id` ( `user_id` ),
KEY `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
break ;
2019-03-19 15:59:31 +01:00
case 32 :
$sqlArray [] = " ALTER TABLE `taxonomies` ADD `required` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-03-19 17:20:46 +01:00
case 33 :
2019-03-05 12:24:56 +01:00
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_publish_kafka` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-06-04 11:30:01 +02:00
case 35 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `notification_logs` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`org_id` int ( 11 ) NOT NULL ,
`type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( `id` ),
KEY `org_id` ( `org_id` ),
KEY `type` ( `type` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2019-09-12 10:22:13 +02:00
break ;
2019-07-08 11:39:41 +02:00
case 36 :
$sqlArray [] = " ALTER TABLE `event_tags` ADD `local` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `attribute_tags` ADD `local` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2019-09-13 11:49:12 +02:00
case 37 :
2019-03-08 09:22:37 +01:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS decaying_models (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
2019-04-03 10:05:45 +02:00
`uuid` varchar ( 40 ) COLLATE utf8_bin DEFAULT NULL ,
2019-09-12 14:44:09 +02:00
`name` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
2019-03-08 09:22:37 +01:00
`parameters` text ,
2019-04-03 11:51:29 +02:00
`attribute_types` text ,
2019-03-08 09:22:37 +01:00
`description` text ,
2019-04-03 10:05:45 +02:00
`org_id` int ( 11 ),
2019-07-24 14:36:13 +02:00
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
2019-08-16 10:33:58 +02:00
`all_orgs` tinyint ( 1 ) NOT NULL DEFAULT 1 ,
2019-04-03 11:51:29 +02:00
`ref` text COLLATE utf8_unicode_ci ,
`formula` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
2019-08-29 12:22:35 +02:00
`version` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
2019-08-30 09:18:00 +02:00
`default` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
2019-09-12 14:28:49 +02:00
PRIMARY KEY ( id ),
INDEX `uuid` ( `uuid` ),
INDEX `name` ( `name` ),
INDEX `org_id` ( `org_id` ),
INDEX `enabled` ( `enabled` ),
INDEX `all_orgs` ( `all_orgs` ),
INDEX `version` ( `version` )
2019-04-03 10:05:45 +02:00
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS decaying_model_mappings (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`attribute_type` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
`model_id` int ( 11 ) NOT NULL ,
2019-09-12 14:28:49 +02:00
PRIMARY KEY ( id ),
INDEX `model_id` ( `model_id` )
2019-03-08 09:22:37 +01:00
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2019-08-20 16:37:06 +02:00
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_decaying` tinyint(1) NOT NULL DEFAULT 0; " ;
2019-08-27 09:19:44 +02:00
$sqlArray [] = " UPDATE `roles` SET `perm_decaying`=1 WHERE `perm_sighting`=1; " ;
2019-07-08 11:39:41 +02:00
break ;
2019-09-13 11:51:05 +02:00
case 38 :
2019-09-13 11:49:12 +02:00
$sqlArray [] = " ALTER TABLE servers ADD priority int(11) NOT NULL DEFAULT 0; " ;
$indexArray [] = array ( 'servers' , 'priority' );
break ;
2019-09-25 11:50:54 +02:00
case 39 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS user_settings (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
2020-04-29 16:13:54 +02:00
`setting` varchar ( 255 ) COLLATE utf8_bin NOT NULL ,
2019-09-25 11:50:54 +02:00
`value` text ,
`user_id` int ( 11 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `key` ( `key` ),
INDEX `user_id` ( `user_id` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2019-09-29 22:20:46 +02:00
case 40 :
$sqlArray [] = " ALTER TABLE `user_settings` ADD `timestamp` int(11) NOT NULL; " ;
$indexArray [] = array ( 'user_settings' , 'timestamp' );
break ;
2019-10-08 11:43:56 +02:00
case 41 :
$sqlArray [] = " ALTER TABLE `roles` ADD `enforce_rate_limit` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `roles` ADD `rate_limit_count` int(11) NOT NULL DEFAULT 0; " ;
break ;
2019-11-06 21:20:04 +01:00
case 42 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS sightingdbs (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 255 ) NOT NULL ,
`description` text ,
`owner` varchar ( 255 ) DEFAULT '' ,
`host` varchar ( 255 ) DEFAULT 'http://localhost' ,
`port` int ( 11 ) DEFAULT 9999 ,
`timestamp` int ( 11 ) NOT NULL ,
`enabled` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`skip_proxy` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`ssl_skip_verification` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` ),
INDEX `owner` ( `owner` ),
INDEX `host` ( `host` ),
INDEX `port` ( `port` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
$sqlArray [] = " CREATE TABLE IF NOT EXISTS sightingdb_orgs (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`sightingdb_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `sightingdb_id` ( `sightingdb_id` ),
INDEX `org_id` ( `org_id` )
) ENGINE = InnoDB ; " ;
break ;
2019-11-08 08:24:04 +01:00
case 43 :
$sqlArray [] = " ALTER TABLE sightingdbs ADD namespace varchar(255) DEFAULT ''; " ;
2019-11-18 11:35:10 +01:00
break ;
case 44 :
$sqlArray [] = " ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1); " ;
2019-11-22 21:53:51 +01:00
break ;
case 45 :
$sqlArray [] = " ALTER TABLE `events` ADD `sighting_timestamp` int(11) NOT NULL DEFAULT 0 AFTER `publish_timestamp`; " ;
$sqlArray [] = " ALTER TABLE `servers` ADD `push_sightings` tinyint(1) NOT NULL DEFAULT 0 AFTER `pull`; " ;
2019-11-08 08:24:04 +01:00
break ;
2020-02-07 12:56:27 +01:00
case 47 :
$this -> __addIndex ( 'tags' , 'numerical_value' );
$this -> __addIndex ( 'taxonomy_predicates' , 'numerical_value' );
$this -> __addIndex ( 'taxonomy_entries' , 'numerical_value' );
break ;
2020-03-08 23:36:27 +01:00
case 49 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS dashboards (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`name` varchar ( 191 ) NOT NULL ,
`description` text ,
`default` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`selectable` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`user_id` int ( 11 ) NOT NULL DEFAULT 0 ,
`restrict_to_org_id` int ( 11 ) NOT NULL DEFAULT 0 ,
`restrict_to_role_id` int ( 11 ) NOT NULL DEFAULT 0 ,
`restrict_to_permission_flag` varchar ( 191 ) NOT NULL DEFAULT '' ,
`value` text ,
`timestamp` int ( 11 ) NOT NULL ,
PRIMARY KEY ( id ),
INDEX `name` ( `name` ),
INDEX `uuid` ( `uuid` ),
INDEX `user_id` ( `user_id` ),
INDEX `restrict_to_org_id` ( `restrict_to_org_id` ),
INDEX `restrict_to_permission_flag` ( `restrict_to_permission_flag` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2020-03-12 10:26:09 +01:00
case 50 :
2020-04-07 13:21:01 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS inbox (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`title` varchar ( 191 ) NOT NULL ,
`type` varchar ( 191 ) NOT NULL ,
`ip` varchar ( 191 ) NOT NULL ,
`user_agent` text ,
`user_agent_sha256` varchar ( 64 ) NOT NULL ,
`comment` text ,
`deleted` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`timestamp` int ( 11 ) NOT NULL ,
`store_as_file` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`data` longtext ,
PRIMARY KEY ( id ),
INDEX `title` ( `title` ),
INDEX `type` ( `type` ),
INDEX `uuid` ( `uuid` ),
INDEX `user_agent_sha256` ( `user_agent_sha256` ),
INDEX `ip` ( `ip` ),
INDEX `timestamp` ( `timestamp` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2020-04-09 14:26:48 +02:00
case 51 :
2020-04-17 14:17:54 +02:00
$sqlArray [] = " ALTER TABLE `feeds` ADD `orgc_id` int(11) NOT NULL DEFAULT 0 " ;
2020-09-01 08:16:17 +02:00
$indexArray [] = array ( 'feeds' , 'orgc_id' );
2020-04-17 14:17:54 +02:00
break ;
2020-04-29 16:13:54 +02:00
case 52 :
if ( ! empty ( $this -> query ( " SHOW COLUMNS FROM `admin_settings` LIKE 'key'; " ))) {
$sqlArray [] = " ALTER TABLE admin_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL; " ;
2020-09-01 08:16:17 +02:00
$indexArray [] = array ( 'admin_settings' , 'setting' );
2020-04-29 16:13:54 +02:00
}
break ;
2020-04-30 06:46:46 +02:00
case 53 :
if ( ! empty ( $this -> query ( " SHOW COLUMNS FROM `user_settings` LIKE 'key'; " ))) {
$sqlArray [] = " ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL; " ;
2020-09-01 08:16:17 +02:00
$indexArray [] = array ( 'user_settings' , 'setting' );
2020-04-30 06:46:46 +02:00
}
break ;
2020-05-14 15:23:28 +02:00
case 54 :
$sqlArray [] = " ALTER TABLE `sightingdbs` MODIFY `timestamp` int(11) NOT NULL DEFAULT 0; " ;
break ;
2020-05-28 14:06:30 +02:00
case 55 :
2020-06-07 19:16:33 +02:00
// index is not used in any SQL query
$this -> __dropIndex ( 'correlations' , 'value' );
// these index can be theoretically used, but probably just in very rare occasion
$this -> __dropIndex ( 'correlations' , 'org_id' );
$this -> __dropIndex ( 'correlations' , 'sharing_group_id' );
$this -> __dropIndex ( 'correlations' , 'a_sharing_group_id' );
2020-06-07 18:54:42 +02:00
break ;
2020-07-01 16:22:55 +02:00
case 56 :
2020-09-01 12:56:35 +02:00
//rename tables
$sqlArray [] = " RENAME TABLE `org_blacklists` TO `org_blocklists`; " ;
$sqlArray [] = " RENAME TABLE `event_blacklists` TO `event_blocklists`; " ;
2020-09-01 15:59:38 +02:00
$sqlArray [] = " RENAME TABLE `whitelist` TO `allowedlist`; " ;
2020-09-01 12:56:35 +02:00
break ;
2020-09-15 12:00:26 +02:00
case 57 :
$sqlArray [] = sprintf ( " INSERT INTO `admin_settings` (`setting`, `value`) VALUES ('fix_login', %s); " , time ());
break ;
2020-09-22 12:08:12 +02:00
case 58 :
2020-10-06 10:52:38 +02:00
$sqlArray [] = " ALTER TABLE `warninglists` MODIFY COLUMN `warninglist_entry_count` int(11) unsigned NOT NULL DEFAULT 0; " ;
break ;
2020-10-09 14:54:47 +02:00
case 59 :
2020-02-18 15:02:35 +01:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS event_reports (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
2020-08-04 10:08:18 +02:00
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
2020-02-18 15:02:35 +01:00
`event_id` int ( 11 ) NOT NULL ,
`name` varchar ( 255 ) NOT NULL ,
`content` text ,
`distribution` tinyint ( 4 ) NOT NULL DEFAULT 0 ,
`sharing_group_id` int ( 11 ),
`timestamp` int ( 11 ) NOT NULL ,
`deleted` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
PRIMARY KEY ( id ),
2020-08-04 10:08:18 +02:00
CONSTRAINT u_uuid UNIQUE ( uuid ),
2020-02-18 15:02:35 +01:00
INDEX `name` ( `name` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2020-10-08 19:27:49 +02:00
case 60 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `attachment_scans` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`type` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`attribute_id` int ( 11 ) NOT NULL ,
`infected` tinyint ( 1 ) NOT NULL ,
`malware_name` varchar ( 191 ) NULL ,
`timestamp` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` ),
INDEX `index` ( `type` , `attribute_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 ; " ;
break ;
2020-11-09 10:07:43 +01:00
case 61 :
2020-10-20 08:35:21 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `auth_keys` (
`id` int ( 10 ) unsigned NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8mb4_unicode_ci NOT NULL ,
`authkey` varchar ( 72 ) CHARACTER SET ascii DEFAULT NULL ,
`authkey_start` varchar ( 4 ) CHARACTER SET ascii DEFAULT NULL ,
`authkey_end` varchar ( 4 ) CHARACTER SET ascii DEFAULT NULL ,
`created` int ( 10 ) unsigned NOT NULL ,
`expiration` int ( 10 ) unsigned NOT NULL ,
`user_id` int ( 10 ) unsigned NOT NULL ,
`comment` text COLLATE utf8mb4_unicode_ci ,
PRIMARY KEY ( `id` ),
KEY `authkey_start` ( `authkey_start` ),
KEY `authkey_end` ( `authkey_end` ),
KEY `created` ( `created` ),
KEY `expiration` ( `expiration` ),
KEY `user_id` ( `user_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
break ;
2020-11-13 16:26:35 +01:00
case 62 :
2020-11-18 11:37:23 +01:00
$sqlArray [] = " ALTER TABLE `auth_keys` MODIFY COLUMN `authkey` varchar(72) CHARACTER SET ascii NOT NULL " ;
$sqlArray [] = " ALTER TABLE `auth_keys` MODIFY COLUMN `authkey_start` varchar(4) CHARACTER SET ascii NOT NULL " ;
$sqlArray [] = " ALTER TABLE `auth_keys` MODIFY COLUMN `authkey_end` varchar(4) CHARACTER SET ascii NOT NULL " ;
$sqlArray [] = " ALTER TABLE `auth_keys` MODIFY COLUMN `comment` text COLLATE utf8mb4_unicode_ci " ;
$sqlArray [] = " ALTER TABLE `attachment_scans` MODIFY COLUMN `malware_name` varchar(191) NULL " ;
break ;
2020-11-18 11:45:01 +01:00
case 63 :
2020-03-12 10:26:09 +01:00
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `distribution` tinyint(4) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `sharing_group_id` int(11); " ;
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `org_id` int(11) NOT NULL; " ;
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `orgc_id` int(11) NOT NULL; " ;
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `default` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `locked` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `extends_uuid` varchar(40) COLLATE utf8_bin DEFAULT ''; " ;
2020-04-29 10:00:22 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `extends_version` int(11) DEFAULT 0; " ;
2020-06-16 16:36:49 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `published` tinyint(1) NOT NULL DEFAULT 0; " ;
2020-07-13 12:34:58 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_clusters` ADD `deleted` TINYINT(1) NOT NULL DEFAULT 0 " ;
2020-03-12 10:26:09 +01:00
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_galaxy_editor` tinyint(1) NOT NULL DEFAULT 0; " ;
2020-07-13 11:14:56 +02:00
2020-03-12 10:26:09 +01:00
$sqlArray [] = " UPDATE `roles` SET `perm_galaxy_editor`=1 WHERE `perm_tag_editor`=1; " ;
2020-05-07 11:03:18 +02:00
$sqlArray [] = " UPDATE `galaxy_clusters` SET `distribution`=3, `default`=1 WHERE `org_id`=0; " ;
2020-07-13 11:14:56 +02:00
2020-05-06 07:59:57 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_reference` RENAME `galaxy_cluster_relations`; " ;
2020-05-25 10:04:07 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_cluster_relations` ADD `galaxy_cluster_uuid` varchar(40) COLLATE utf8_bin NOT NULL; " ;
2020-05-07 11:03:18 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_cluster_relations` ADD `distribution` tinyint(4) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `galaxy_cluster_relations` ADD `sharing_group_id` int(11); " ;
$sqlArray [] = " ALTER TABLE `galaxy_cluster_relations` ADD `default` tinyint(1) NOT NULL DEFAULT 0; " ;
2020-06-15 08:31:40 +02:00
$sqlArray [] = " ALTER TABLE `galaxy_cluster_relations` DROP COLUMN `referenced_galaxy_cluster_value`; " ;
2020-05-06 07:59:57 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `galaxy_cluster_relation_tags` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`galaxy_cluster_relation_id` int ( 11 ) NOT NULL ,
`tag_id` int ( 11 ) NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
2020-07-13 11:14:56 +02:00
2020-06-18 11:02:08 +02:00
$sqlArray [] = " ALTER TABLE `tags` ADD `is_galaxy` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `tags` ADD `is_custom_galaxy` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " UPDATE `tags` SET `is_galaxy`=1 WHERE `name` LIKE 'misp-galaxy:%'; " ;
$sqlArray [] = " UPDATE `tags` SET `is_custom_galaxy`=1 WHERE `name` REGEXP '^misp-galaxy:[^:= \" ]+= \" [a-fA-F0-9] { 8}-[a-fA-F0-9] { 4}-[a-fA-F0-9] { 4}-[a-fA-F0-9] { 4}-[a-fA-F0-9] { 12} \" $ '; " ;
2020-07-13 11:14:56 +02:00
2020-05-26 15:08:24 +02:00
$sqlArray [] = " ALTER TABLE `servers` ADD `push_galaxy_clusters` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_sightings`; " ;
2020-06-10 09:36:34 +02:00
$sqlArray [] = " ALTER TABLE `servers` ADD `pull_galaxy_clusters` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_galaxy_clusters`; " ;
2020-07-13 11:14:56 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `galaxy_cluster_blocklists` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`cluster_uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`created` datetime NOT NULL ,
`cluster_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL ,
`comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci ,
`cluster_orgc` VARCHAR ( 255 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL ,
PRIMARY KEY ( `id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COLLATE = utf8_bin ; " ;
2020-05-28 14:17:23 +02:00
$indexArray [] = array ( 'galaxy_clusters' , 'org_id' );
$indexArray [] = array ( 'galaxy_clusters' , 'orgc_id' );
$indexArray [] = array ( 'galaxy_clusters' , 'sharing_group_id' );
$indexArray [] = array ( 'galaxy_clusters' , 'extends_uuid' );
$indexArray [] = array ( 'galaxy_clusters' , 'extends_version' );
$indexArray [] = array ( 'galaxy_clusters' , 'default' );
$indexArray [] = array ( 'galaxy_cluster_relations' , 'galaxy_cluster_uuid' );
$indexArray [] = array ( 'galaxy_cluster_relations' , 'sharing_group_id' );
$indexArray [] = array ( 'galaxy_cluster_relations' , 'default' );
$indexArray [] = array ( 'galaxy_cluster_relation_tags' , 'galaxy_cluster_relation_id' );
$indexArray [] = array ( 'galaxy_cluster_relation_tags' , 'tag_id' );
2020-07-13 11:14:56 +02:00
$indexArray [] = array ( 'galaxy_cluster_blocklists' , 'cluster_uuid' );
$indexArray [] = array ( 'galaxy_cluster_blocklists' , 'cluster_orgc' );
2020-03-12 10:26:09 +01:00
break ;
2020-11-30 23:38:38 +01:00
case 64 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `cerebrates` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`name` varchar ( 191 ) NOT NULL ,
`url` varchar ( 255 ) NOT NULL ,
`authkey` varchar ( 40 ) CHARACTER SET ascii COLLATE ascii_general_ci NULL ,
`open` tinyint ( 1 ) DEFAULT 0 ,
`org_id` int ( 11 ) NOT NULL ,
`pull_orgs` tinyint ( 1 ) DEFAULT 0 ,
`pull_sharing_groups` tinyint ( 1 ) DEFAULT 0 ,
`self_signed` tinyint ( 1 ) DEFAULT 0 ,
`cert_file` varchar ( 255 ) DEFAULT NULL ,
`client_cert_file` varchar ( 255 ) DEFAULT NULL ,
`internal` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`skip_proxy` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`description` text ,
PRIMARY KEY ( `id` ),
KEY `url` ( `url` ),
KEY `org_id` ( `org_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
break ;
2021-01-07 09:31:38 +01:00
case 65 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `correlation_exclusions` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`value` text NOT NULL ,
`from_json` tinyint ( 1 ) default 0 ,
PRIMARY KEY ( `id` ),
2021-11-04 11:04:56 +01:00
UNIQUE INDEX `value` ( `value` ( 191 ))
2021-01-07 09:31:38 +01:00
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
break ;
2021-02-19 13:33:24 +01:00
case 66 :
2021-02-22 10:33:16 +01:00
$sqlArray [] = " ALTER TABLE `galaxy_clusters` MODIFY COLUMN `tag_name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT ''; " ;
2021-02-22 13:56:30 +01:00
$indexArray [] = [ 'event_reports' , 'event_id' ];
2021-02-22 10:33:16 +01:00
break ;
2021-02-27 11:13:47 +01:00
case 67 :
$sqlArray [] = " ALTER TABLE `auth_keys` ADD `allowed_ips` text DEFAULT NULL; " ;
2021-04-27 10:52:53 +02:00
break ;
case 68 :
$sqlArray [] = " ALTER TABLE `correlation_exclusions` ADD `comment` text DEFAULT NULL; " ;
2021-01-22 13:01:23 +01:00
break ;
case 69 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `audit_logs` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`created` datetime NOT NULL ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`authkey_id` int ( 11 ) DEFAULT NULL ,
`ip` varbinary ( 16 ) DEFAULT NULL ,
`request_type` tinyint NOT NULL ,
`request_id` varchar ( 255 ) DEFAULT NULL ,
`action` varchar ( 20 ) NOT NULL ,
`model` varchar ( 80 ) NOT NULL ,
`model_id` int ( 11 ) NOT NULL ,
`model_title` text DEFAULT NULL ,
`event_id` int ( 11 ) NULL ,
`change` blob ,
PRIMARY KEY ( `id` ),
INDEX `event_id` ( `event_id` ),
INDEX `model_id` ( `model_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
2021-06-03 11:01:49 +02:00
break ;
case 70 :
2021-02-19 13:33:24 +01:00
$sqlArray [] = " ALTER TABLE `galaxies` ADD `enabled` tinyint(1) NOT NULL DEFAULT 1 AFTER `namespace`; " ;
2021-06-01 17:49:45 +02:00
break ;
case 71 :
$sqlArray [] = " ALTER TABLE `roles` ADD `perm_warninglist` tinyint(1) NOT NULL DEFAULT 0; " ;
$sqlArray [] = " ALTER TABLE `warninglist_entries` ADD `comment` text DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `warninglists` ADD `default` tinyint(1) NOT NULL DEFAULT 1, ADD `category` varchar(20) NOT NULL DEFAULT 'false_positive', DROP COLUMN `warninglist_entry_count` " ;
2021-06-08 15:19:49 +02:00
break ;
case 72 :
$sqlArray [] = " ALTER TABLE `auth_keys` ADD `read_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `expiration`; " ;
2021-02-19 13:33:24 +01:00
break ;
2021-09-06 12:10:35 +02:00
case 73 :
$this -> __dropIndex ( 'user_settings' , 'timestamp' ); // index is not used
$sqlArray [] = " ALTER TABLE `user_settings` ADD UNIQUE INDEX `unique_setting` (`user_id`, `setting`) " ;
break ;
2021-10-23 19:07:29 +02:00
case 74 :
$sqlArray [] = " ALTER TABLE `users` MODIFY COLUMN `change_pw` tinyint(1) NOT NULL DEFAULT 0; " ;
break ;
2021-10-24 13:27:11 +02:00
case 75 :
$this -> __addIndex ( 'object_references' , 'event_id' );
$this -> __dropIndex ( 'object_references' , 'timestamp' );
$this -> __dropIndex ( 'object_references' , 'source_uuid' );
$this -> __dropIndex ( 'object_references' , 'relationship_type' );
$this -> __dropIndex ( 'object_references' , 'referenced_uuid' );
break ;
2021-11-03 17:09:42 +01:00
case 76 :
2021-11-04 16:28:41 +01:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `system_settings` (
`setting` varchar ( 255 ) NOT NULL ,
`value` blob NOT NULL ,
PRIMARY KEY ( `setting` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
2021-11-07 16:58:42 +01:00
$sqlArray [] = " ALTER TABLE `servers` MODIFY COLUMN `authkey` VARBINARY(255) NOT NULL; " ;
$sqlArray [] = " ALTER TABLE `cerebrates` MODIFY COLUMN `authkey` VARBINARY(255) NOT NULL; " ;
2021-11-04 16:28:41 +01:00
break ;
2021-11-08 11:35:20 +01:00
case 77 :
2021-10-27 16:46:56 +02:00
$sqlArray [] = " ALTER TABLE `tags` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `is_custom_galaxy`; " ;
$sqlArray [] = " ALTER TABLE `galaxies` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `enabled`; " ;
break ;
2021-11-08 14:37:27 +01:00
case 78 :
2021-11-04 16:51:56 +01:00
$sqlArray [] = " ALTER TABLE `jobs` MODIFY COLUMN `process_id` varchar(36) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL; " ;
2021-11-11 15:47:30 +01:00
break ;
case 79 :
$sqlArray [] = " ALTER TABLE `users` ADD `sub` varchar(255) NULL DEFAULT NULL; " ;
$sqlArray [] = " ALTER TABLE `users` ADD UNIQUE INDEX `sub` (`sub`); " ;
break ;
2022-03-02 02:09:20 +01:00
case 80 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `sharing_group_blueprints` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`name` varchar ( 191 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`sharing_group_id` int ( 11 ),
`rules` text ,
PRIMARY KEY ( `id` ),
INDEX `uuid` ( `uuid` ),
INDEX `name` ( `name` ),
INDEX `org_id` ( `org_id` ),
INDEX `sharing_group_id` ( `sharing_group_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
break ;
2022-03-02 17:32:35 +01:00
case 81 :
2022-03-04 16:10:12 +01:00
$fields = [ 'nationality' , 'sector' , 'type' , 'name' ];
2022-03-02 17:32:35 +01:00
foreach ( $fields as $field ) {
$sqlArray [] = sprintf ( " UPDATE organisations SET %s = '' WHERE %s IS NULL; " , $field , $field );
$sqlArray [] = sprintf ( " ALTER table organisations MODIFY %s varchar(255) NOT NULL DEFAULT ''; " , $field );
}
break ;
2022-03-04 16:10:12 +01:00
case 82 :
$sqlArray [] = sprintf ( " ALTER table organisations MODIFY description text; " );
break ;
2022-03-13 12:35:12 +01:00
case 83 :
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `sharing_group_blueprints` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`name` varchar ( 191 ) NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`user_id` int ( 11 ) NOT NULL ,
`org_id` int ( 11 ) NOT NULL ,
`sharing_group_id` int ( 11 ),
`rules` text ,
PRIMARY KEY ( `id` ),
INDEX `uuid` ( `uuid` ),
INDEX `name` ( `name` ),
INDEX `org_id` ( `org_id` ),
INDEX `sharing_group_id` ( `sharing_group_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
break ;
case 84 :
$sqlArray [] = sprintf ( " ALTER table events add `protected` tinyint(1); " );
$sqlArray [] = " CREATE TABLE IF NOT EXISTS `cryptographic_keys` (
`id` int ( 11 ) NOT NULL AUTO_INCREMENT ,
`uuid` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`type` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`timestamp` int ( 11 ) NOT NULL DEFAULT 0 ,
`parent_id` int ( 11 ) NOT NULL ,
`parent_type` varchar ( 40 ) COLLATE utf8_bin NOT NULL ,
`key_data` text ,
`revoked` tinyint ( 1 ) NOT NULL DEFAULT 0 ,
`fingerprint` varchar ( 255 ) COLLATE utf8_bin NOT NULL DEFAULT '' ,
PRIMARY KEY ( `id` ),
INDEX `uuid` ( `uuid` ),
INDEX `type` ( `type` ),
INDEX `parent_id` ( `parent_id` )
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci ; " ;
break ;
2022-03-17 14:43:01 +01:00
case 85 :
$this -> __addIndex ( 'cryptographic_keys' , 'parent_type' );
$this -> __addIndex ( 'cryptographic_keys' , 'fingerprint' );
break ;
2022-05-04 09:04:30 +02:00
case 86 :
$this -> __addIndex ( 'attributes' , 'timestamp' );
2022-05-04 09:22:01 +02:00
break ;
2018-07-19 11:48:22 +02:00
case 'fixNonEmptySharingGroupID' :
$sqlArray [] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;' ;
$sqlArray [] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;' ;
break ;
case 'cleanupAfterUpgrade' :
$sqlArray [] = 'ALTER TABLE `events` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `events` DROP `orgc`;' ;
$sqlArray [] = 'ALTER TABLE `correlations` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `jobs` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `servers` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `servers` DROP `organization`;' ;
$sqlArray [] = 'ALTER TABLE `shadow_attributes` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `shadow_attributes` DROP `event_org`;' ;
$sqlArray [] = 'ALTER TABLE `threads` DROP `org`;' ;
$sqlArray [] = 'ALTER TABLE `users` DROP `org`;' ;
break ;
2019-06-13 09:16:34 +02:00
case 'seenOnAttributeAndObject' :
$sqlArray [] =
" ALTER TABLE `attributes`
DROP INDEX uuid ,
DROP INDEX event_id ,
DROP INDEX sharing_group_id ,
DROP INDEX type ,
DROP INDEX category ,
DROP INDEX value1 ,
DROP INDEX value2 ,
DROP INDEX object_id ,
2019-06-13 09:38:39 +02:00
DROP INDEX object_relation ;
" ;
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX deleted " ; // deleted index may not be present
2019-07-02 16:26:05 +02:00
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX comment " ; // for replayability
2019-06-13 09:38:39 +02:00
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX first_seen " ; // for replayability
$sqlArray [] = " ALTER TABLE `attributes` DROP INDEX last_seen " ; // for replayability
2019-06-13 09:16:34 +02:00
$sqlArray [] =
" ALTER TABLE `attributes`
ADD COLUMN `first_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
ADD COLUMN `last_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
MODIFY comment TEXT COLLATE utf8_unicode_ci
; " ;
2019-07-01 14:35:56 +02:00
$indexArray [] = array ( 'attributes' , 'uuid' );
$indexArray [] = array ( 'attributes' , 'event_id' );
$indexArray [] = array ( 'attributes' , 'sharing_group_id' );
$indexArray [] = array ( 'attributes' , 'type' );
$indexArray [] = array ( 'attributes' , 'category' );
2019-12-16 14:59:53 +01:00
$indexArray [] = array ( 'attributes' , 'value1' , 255 );
$indexArray [] = array ( 'attributes' , 'value2' , 255 );
2019-07-01 14:35:56 +02:00
$indexArray [] = array ( 'attributes' , 'object_id' );
$indexArray [] = array ( 'attributes' , 'object_relation' );
$indexArray [] = array ( 'attributes' , 'deleted' );
$indexArray [] = array ( 'attributes' , 'first_seen' );
$indexArray [] = array ( 'attributes' , 'last_seen' );
2019-06-13 09:16:34 +02:00
$sqlArray [] = "
ALTER TABLE `objects`
ADD `first_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
2019-07-02 16:26:05 +02:00
ADD `last_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
MODIFY comment TEXT COLLATE utf8_unicode_ci
2019-06-13 09:16:34 +02:00
; " ;
$indexArray [] = array ( 'objects' , 'first_seen' );
$indexArray [] = array ( 'objects' , 'last_seen' );
2019-06-24 10:28:55 +02:00
$sqlArray [] = "
ALTER TABLE `shadow_attributes`
ADD `first_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
2019-07-02 16:26:05 +02:00
ADD `last_seen` BIGINT ( 20 ) NULL DEFAULT NULL ,
MODIFY comment TEXT COLLATE utf8_unicode_ci
2019-06-24 10:28:55 +02:00
; " ;
$indexArray [] = array ( 'shadow_attributes' , 'first_seen' );
$indexArray [] = array ( 'shadow_attributes' , 'last_seen' );
2019-06-13 09:16:34 +02:00
break ;
2018-07-19 11:48:22 +02:00
default :
return false ;
}
2019-04-26 09:45:03 +02:00
// switch MISP instance live to false
if ( $liveOff ) {
2021-09-30 15:13:34 +02:00
$this -> setLive ( false );
2019-04-26 09:45:03 +02:00
}
2019-04-29 11:09:04 +02:00
$sql_update_count = count ( $sqlArray );
$index_update_count = count ( $indexArray );
$total_update_count = $sql_update_count + $index_update_count ;
2019-10-03 13:50:55 +02:00
$this -> __setUpdateProgress ( 0 , $total_update_count , $command );
2019-04-29 11:09:04 +02:00
$str_index_array = array ();
2019-04-26 09:45:03 +02:00
foreach ( $indexArray as $toIndex ) {
2020-11-28 15:14:31 +01:00
$str_index_array [] = __ ( 'Indexing %s -> %s' , $toIndex [ 0 ], $toIndex [ 1 ]);
2019-04-26 09:45:03 +02:00
}
2019-04-29 11:09:04 +02:00
$this -> __setUpdateCmdMessages ( array_merge ( $sqlArray , $str_index_array ));
2019-10-29 09:57:25 +01:00
$flagStop = false ;
$errorCount = 0 ;
2019-04-29 11:26:55 +02:00
// execute test before update. Exit if it fails
if ( isset ( $this -> advanced_updates_description [ $command ][ 'preUpdate' ])) {
$function_name = $this -> advanced_updates_description [ $command ][ 'preUpdate' ];
2018-07-19 11:48:22 +02:00
try {
2019-04-29 11:26:55 +02:00
$this -> { $function_name }();
} catch ( Exception $e ) {
$this -> __setPreUpdateTestState ( false );
$this -> __setUpdateProgress ( 0 , false );
2021-09-30 15:13:34 +02:00
$this -> __setUpdateResMessages ( 0 , __ ( 'Issues executing the pre-update test `%s`. The returned error is: %s' , $function_name , $e -> getMessage ()) . PHP_EOL );
2019-04-29 11:26:55 +02:00
$this -> __setUpdateError ( 0 );
2019-10-29 09:57:25 +01:00
$errorCount ++ ;
2019-04-29 11:26:55 +02:00
$exitOnError = true ;
2019-10-29 09:57:25 +01:00
$flagStop = true ;
2019-04-29 11:26:55 +02:00
}
}
2019-10-29 09:57:25 +01:00
if ( ! $flagStop ) {
2019-04-29 11:26:55 +02:00
$this -> __setPreUpdateTestState ( true );
foreach ( $sqlArray as $i => $sql ) {
try {
$this -> __setUpdateProgress ( $i , false );
$this -> query ( $sql );
$this -> Log -> create ();
$this -> Log -> save ( array (
2018-07-19 11:48:22 +02:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
2020-11-28 15:14:31 +01:00
'title' => __ ( 'Successfully executed the SQL query for ' ) . $command ,
2021-09-30 15:13:34 +02:00
'change' => __ ( 'The executed SQL query was: %s' , $sql ),
2019-04-29 11:26:55 +02:00
));
2021-09-30 15:13:34 +02:00
$this -> __setUpdateResMessages ( $i , __ ( 'Successfully executed the SQL query for %s' , $command ));
2019-04-29 11:26:55 +02:00
} catch ( Exception $e ) {
2019-10-29 09:57:25 +01:00
$errorMessage = $e -> getMessage ();
2019-04-29 11:26:55 +02:00
$this -> Log -> create ();
2019-10-29 09:57:25 +01:00
$logMessage = array (
2018-07-19 11:48:22 +02:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
2021-09-30 15:13:34 +02:00
'title' => __ ( 'Issues executing the SQL query for %s' , $command ),
2019-10-29 09:57:25 +01:00
'change' => __ ( 'The executed SQL query was: ' ) . $sql . PHP_EOL . __ ( ' The returned error is: ' ) . $errorMessage
2019-09-30 10:30:41 +02:00
);
2022-05-14 10:17:06 +02:00
$this -> __setUpdateResMessages ( $i , __ ( 'Issues executing the SQL query for `%s`. The returned error is: ' . PHP_EOL . '%s' , $command , $errorMessage ));
if ( ! $this -> isAcceptedDatabaseError ( $errorMessage )) {
2019-08-14 12:13:45 +02:00
$this -> __setUpdateError ( $i );
2019-10-29 09:57:25 +01:00
$errorCount ++ ;
2019-08-14 12:13:45 +02:00
if ( $exitOnError ) {
2019-10-29 09:57:25 +01:00
$flagStop = true ;
2019-08-14 12:13:45 +02:00
break ;
}
2019-09-30 10:30:41 +02:00
} else {
2020-09-01 15:59:38 +02:00
$logMessage [ 'change' ] = $logMessage [ 'change' ] . PHP_EOL . __ ( 'However, as this error is allowed, the update went through.' );
2019-04-29 11:26:55 +02:00
}
2019-10-29 09:57:25 +01:00
$this -> Log -> save ( $logMessage );
2018-07-19 11:48:22 +02:00
}
}
}
2019-10-29 09:57:25 +01:00
if ( ! $flagStop ) {
2019-09-30 10:30:41 +02:00
if ( ! empty ( $indexArray )) {
if ( $clean ) {
$this -> cleanCacheFiles ();
}
foreach ( $indexArray as $i => $iA ) {
$this -> __setUpdateProgress ( count ( $sqlArray ) + $i , false );
if ( isset ( $iA [ 2 ])) {
2019-12-16 14:59:53 +01:00
$indexSuccess = $this -> __addIndex ( $iA [ 0 ], $iA [ 1 ], $iA [ 2 ]);
2019-09-30 10:30:41 +02:00
} else {
2019-12-16 14:59:53 +01:00
$indexSuccess = $this -> __addIndex ( $iA [ 0 ], $iA [ 1 ]);
}
if ( $indexSuccess [ 'success' ]) {
2021-09-30 15:13:34 +02:00
$this -> __setUpdateResMessages ( count ( $sqlArray ) + $i , __ ( 'Successfully indexed %s -> %s' , $iA [ 0 ], $iA [ 1 ]));
2019-12-16 14:59:53 +01:00
} else {
2020-01-20 10:23:51 +01:00
$this -> __setUpdateResMessages ( count ( $sqlArray ) + $i , sprintf ( '%s %s %s %s' ,
2019-12-16 14:59:53 +01:00
__ ( 'Failed to add index' ),
sprintf ( '%s -> %s' , $iA [ 0 ], $iA [ 1 ]),
__ ( 'The returned error is:' ) . PHP_EOL ,
$indexSuccess [ 'errorMessage' ]
));
$this -> __setUpdateError ( count ( $sqlArray ) + $i );
2019-09-30 10:30:41 +02:00
}
}
}
2020-11-28 15:14:31 +01:00
$this -> __setUpdateProgress ( count ( $sqlArray ) + count ( $indexArray ), false );
2019-04-26 09:45:03 +02:00
}
2018-07-19 11:48:22 +02:00
if ( $clean ) {
$this -> cleanCacheFiles ();
}
2019-04-26 09:45:03 +02:00
if ( $liveOff ) {
2021-09-30 15:13:34 +02:00
$this -> setLive ( true );
2019-04-26 09:45:03 +02:00
}
2019-10-29 09:57:25 +01:00
if ( ! $flagStop && $errorCount == 0 ) {
2019-04-26 09:45:03 +02:00
$this -> __postUpdate ( $command );
}
2019-10-29 09:57:25 +01:00
if ( $flagStop && $errorCount > 0 ) {
2019-08-14 12:13:45 +02:00
$this -> Log -> create ();
$this -> Log -> save ( array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => __ ( 'Issues executing the SQL query for %s' , $command ),
'change' => __ ( 'Database updates stopped as some errors occurred and the stop flag is enabled.' )
2019-08-14 12:13:45 +02:00
));
return false ;
}
2018-07-19 11:48:22 +02:00
return true ;
}
2021-09-30 15:13:34 +02:00
/**
* Set if misp is live in redis or in config file as fallback
* @ param bool $isLive
*/
private function setLive ( $isLive )
{
try {
$redis = $this -> setupRedisWithException ();
if ( $isLive ) {
$redis -> del ( 'misp:live' );
} else {
$redis -> set ( 'misp:live' , '0' );
}
} catch ( Exception $e ) {
// pass
}
if ( ! isset ( $this -> Server )) {
$this -> Server = ClassRegistry :: init ( 'Server' );
}
$this -> Server -> serverSettingsSaveValue ( 'MISP.live' , $isLive );
}
2019-04-26 09:45:03 +02:00
// check whether the adminSetting should be updated after the update
private function __postUpdate ( $command ) {
if ( isset ( $this -> advanced_updates_description [ $command ][ 'record' ])) {
if ( $this -> advanced_updates_description [ $command ][ 'record' ]) {
$this -> AdminSetting -> changeSetting ( $command , 1 );
}
}
}
2018-07-19 11:48:22 +02:00
private function __dropIndex ( $table , $field )
{
$this -> Log = ClassRegistry :: init ( 'Log' );
$indexCheckResult = array ();
2022-05-14 10:17:06 +02:00
if ( $this -> isMysql ()) {
2018-07-19 11:48:22 +02:00
$indexCheck = " SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name=' " . $table . " ' AND index_name LIKE ' " . $field . " %'; " ;
$indexCheckResult = $this -> query ( $indexCheck );
2022-05-14 10:17:06 +02:00
} else {
2018-07-19 11:48:22 +02:00
$pgIndexName = 'idx_' . $table . '_' . $field ;
$indexCheckResult [] = array ( 'STATISTICS' => array ( 'INDEX_NAME' => $pgIndexName ));
}
foreach ( $indexCheckResult as $icr ) {
2022-05-14 10:17:06 +02:00
if ( $this -> isMysql ()) {
2018-07-19 11:48:22 +02:00
$dropIndex = 'ALTER TABLE ' . $table . ' DROP INDEX ' . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ';' ;
2022-05-14 10:17:06 +02:00
} else {
2018-07-19 11:48:22 +02:00
$dropIndex = 'DROP INDEX IF EXISTS ' . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ';' ;
}
$result = true ;
try {
$this -> query ( $dropIndex );
} catch ( Exception $e ) {
$result = false ;
}
$this -> Log -> create ();
$this -> Log -> save ( array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => ( $result ? 'Removed index ' : 'Failed to remove index ' ) . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ' from ' . $table ,
'change' => ( $result ? 'Removed index ' : 'Failed to remove index ' ) . $icr [ 'STATISTICS' ][ 'INDEX_NAME' ] . ' from ' . $table ,
2018-07-19 11:48:22 +02:00
));
}
}
2021-02-22 13:56:30 +01:00
private function __addIndex ( $table , $field , $length = null , $unique = false )
2018-07-19 11:48:22 +02:00
{
$this -> Log = ClassRegistry :: init ( 'Log' );
2021-02-22 13:56:30 +01:00
$index = $unique ? 'UNIQUE INDEX' : 'INDEX' ;
2022-05-14 10:17:06 +02:00
if ( ! $this -> isMysql ()) {
2021-02-22 13:56:30 +01:00
$addIndex = " CREATE $index idx_ " . $table . " _ " . $field . " ON " . $table . " ( " . $field . " ); " ;
2018-07-19 11:48:22 +02:00
} else {
if ( ! $length ) {
2021-02-22 13:56:30 +01:00
$addIndex = " ALTER TABLE ` " . $table . " ` ADD $index ` " . $field . " ` (` " . $field . " `); " ;
2018-07-19 11:48:22 +02:00
} else {
2021-02-22 13:56:30 +01:00
$addIndex = " ALTER TABLE ` " . $table . " ` ADD $index ` " . $field . " ` (` " . $field . " `( " . $length . " )); " ;
2018-07-19 11:48:22 +02:00
}
}
$result = true ;
$duplicate = false ;
2019-12-16 14:59:53 +01:00
$errorMessage = '' ;
2018-07-19 11:48:22 +02:00
try {
$this -> query ( $addIndex );
} catch ( Exception $e ) {
2021-02-22 13:56:30 +01:00
$duplicate = strpos ( $e -> getMessage (), '1061' ) !== false ;
2019-12-16 14:59:53 +01:00
$errorMessage = $e -> getMessage ();
2018-07-19 11:48:22 +02:00
$result = false ;
}
$this -> Log -> create ();
$this -> Log -> save ( array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => ( $result ? 'Added index ' : 'Failed to add index ' ) . $field . ' to ' . $table . ( $duplicate ? ' (index already set)' : $errorMessage ),
'change' => ( $result ? 'Added index ' : 'Failed to add index ' ) . $field . ' to ' . $table . ( $duplicate ? ' (index already set)' : $errorMessage ),
2018-07-19 11:48:22 +02:00
));
2019-12-16 14:59:53 +01:00
$additionResult = array ( 'success' => $result || $duplicate );
if ( ! $result ) {
$additionResult [ 'errorMessage' ] = $errorMessage ;
}
return $additionResult ;
2018-07-19 11:48:22 +02:00
}
public function cleanCacheFiles ()
{
Cache :: clear ();
2019-09-16 19:41:27 +02:00
Cache :: clear ( false , '_cake_core_' );
Cache :: clear ( false , '_cake_model_' );
2018-07-19 11:48:22 +02:00
clearCache ();
2019-09-16 19:41:27 +02:00
$files = glob ( CACHE . 'models' . DS . 'myapp*' );
2018-07-19 11:48:22 +02:00
$files = array_merge ( $files , glob ( CACHE . 'persistent' . DS . 'myapp*' ));
2019-09-16 19:41:27 +02:00
foreach ( $files as $file ) {
if ( is_file ( $file )) {
unlink ( $file );
2018-07-19 11:48:22 +02:00
}
}
2021-08-02 13:30:29 +02:00
return true ;
2018-07-19 11:48:22 +02:00
}
2018-08-20 10:50:09 +02:00
public function validateAuthkey ( $value )
{
2018-08-03 18:40:46 +02:00
if ( empty ( $value [ 'authkey' ])) {
return 'Empty authkey found. Make sure you set the 40 character long authkey.' ;
}
if ( ! preg_match ( '/[a-z0-9]{40}/i' , $value [ 'authkey' ])) {
return 'The authkey has to be exactly 40 characters long and consist of alphanumeric characters.' ;
}
return true ;
}
2018-07-19 11:48:22 +02:00
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty ( $value )
{
2021-10-17 16:49:26 +02:00
$field = array_keys ( $value )[ 0 ];
2021-10-17 14:21:47 +02:00
$value = trim ( $value [ $field ]);
if ( ! empty ( $value )) {
2018-07-19 11:48:22 +02:00
return true ;
}
return ucfirst ( $field ) . ' cannot be empty.' ;
}
2019-09-29 20:20:39 +02:00
public function valueIsJson ( $value )
{
2021-10-17 16:49:26 +02:00
$value = array_values ( $value )[ 0 ];
2021-10-17 14:21:47 +02:00
$json_decoded = json_decode ( $value );
2019-09-29 20:20:39 +02:00
if ( $json_decoded === null ) {
return __ ( 'Invalid JSON.' );
}
return true ;
}
2020-03-12 10:26:09 +01:00
2018-07-19 11:48:22 +02:00
public function valueIsID ( $value )
{
2021-10-17 16:49:26 +02:00
$field = array_keys ( $value )[ 0 ];
2018-07-19 11:48:22 +02:00
if ( ! is_numeric ( $value [ $field ]) || $value [ $field ] < 0 ) {
return 'Invalid ' . ucfirst ( $field ) . ' ID' ;
}
return true ;
}
public function stringNotEmpty ( $value )
{
2021-10-17 16:49:26 +02:00
$field = array_keys ( $value )[ 0 ];
2021-10-17 14:21:47 +02:00
$value = trim ( $value [ $field ]);
if ( ! isset ( $value ) || ( $value == false && $value !== " 0 " )) {
2018-07-19 11:48:22 +02:00
return ucfirst ( $field ) . ' cannot be empty.' ;
}
return true ;
}
2019-06-13 09:16:34 +02:00
// Try to create a table with a BIGINT(20)
2021-11-07 10:40:01 +01:00
public function seenOnAttributeAndObjectPreUpdate ()
{
2019-06-13 09:16:34 +02:00
$sqlArray [] = " CREATE TABLE IF NOT EXISTS testtable (
`testfield` BIGINT ( 6 ) NULL DEFAULT NULL
) ENGINE = InnoDB DEFAULT CHARSET = utf8 ; " ;
try {
foreach ( $sqlArray as $i => $sql ) {
$this -> query ( $sql );
}
} catch ( Exception $e ) {
throw new Exception ( 'Pre update test failed: ' . PHP_EOL . $sql . PHP_EOL . ' The returned error is: ' . $e -> getMessage ());
}
// clean up
$sqlArray [] = " DROP TABLE testtable; " ;
foreach ( $sqlArray as $i => $sql ) {
$this -> query ( $sql );
}
}
2019-09-30 15:31:49 +02:00
public function runUpdates ( $verbose = false , $useWorker = true , $processId = false )
2018-07-19 11:48:22 +02:00
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
2019-09-30 16:28:52 +02:00
$this -> Job = ClassRegistry :: init ( 'Job' );
2021-10-16 09:58:05 +02:00
2018-07-19 11:48:22 +02:00
$db = ConnectionManager :: getDataSource ( 'default' );
$tables = $db -> listSources ();
$requiresLogout = false ;
// if we don't even have an admin table, time to create it.
2021-09-26 21:36:22 +02:00
if ( ! in_array ( 'admin_settings' , $tables , true )) {
2018-07-19 11:48:22 +02:00
$this -> updateDatabase ( 'adminTable' );
$requiresLogout = true ;
} else {
$this -> __runCleanDB ();
2021-10-16 09:58:05 +02:00
$db_version = $this -> AdminSetting -> find ( 'all' , [
'conditions' => array ( 'setting' => 'db_version' ),
'fields' => [ 'id' , 'value' ],
]);
2018-07-19 11:48:22 +02:00
if ( count ( $db_version ) > 1 ) {
// we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
foreach ( $db_version as $k => $v ) {
if ( $k > 0 ) {
$this -> AdminSetting -> delete ( $v [ 'AdminSetting' ][ 'id' ]);
}
}
}
$db_version = $db_version [ 0 ];
2021-02-25 11:37:37 +01:00
$updates = $this -> findUpgrades ( $db_version [ 'AdminSetting' ][ 'value' ]);
2020-11-19 19:16:12 +01:00
if ( $processId ) {
$job = $this -> Job -> find ( 'first' , array (
'conditions' => array ( 'Job.id' => $processId )
));
} else {
$job = null ;
}
2020-01-20 10:25:00 +01:00
if ( ! empty ( $updates )) {
2021-10-16 09:58:05 +02:00
$this -> Log = ClassRegistry :: init ( 'Log' );
$this -> Server = ClassRegistry :: init ( 'Server' );
2019-09-30 16:28:52 +02:00
// Exit if updates are locked.
// This is not as reliable as a real lock implementation
// However, as all updates are re-playable, there is no harm if they
// get played multiple time. The purpose of this lightweight lock
// is only to limit the load.
if ( $this -> isUpdateLocked ()) { // prevent creation of useless workers
$this -> Log -> create ();
$this -> Log -> save ( array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_db_worker' ,
'user_id' => 0 ,
'title' => __ ( 'Issues executing run_updates' ),
'change' => __ ( 'Database updates are locked. Worker not spawned' )
2019-09-30 16:28:52 +02:00
));
2019-10-01 13:08:09 +02:00
if ( ! empty ( $job )) { // if multiple prio worker is enabled, want to mark them as done
$job [ 'Job' ][ 'progress' ] = 100 ;
$job [ 'Job' ][ 'message' ] = __ ( 'Update done' );
$this -> Job -> save ( $job );
}
return true ;
2019-09-30 16:28:52 +02:00
}
2019-10-01 13:08:09 +02:00
2019-05-08 15:37:39 +02:00
// restart this function by a worker
if ( $useWorker && Configure :: read ( 'MISP.background_jobs' )) {
2019-10-01 13:36:37 +02:00
$workerIssueCount = 0 ;
$workerDiagnostic = $this -> Server -> workerDiagnostics ( $workerIssueCount );
if ( isset ( $workerDiagnostic [ 'update' ][ 'ok' ]) && $workerDiagnostic [ 'update' ][ 'ok' ]) {
$workerType = 'update' ;
2020-10-01 09:28:02 +02:00
} else { // update worker not running, doing the update inline
2019-10-01 13:36:37 +02:00
return $this -> runUpdates ( $verbose , false );
}
2021-11-02 11:55:12 +01:00
2021-11-02 16:53:42 +01:00
/** @var Job $job */
$job = ClassRegistry :: init ( 'Job' );
$jobId = $job -> createJob (
'SYSTEM' ,
Job :: WORKER_PRIO ,
'run_updates' ,
'command: ' . implode ( ',' , $updates ),
'Updating.'
);
2021-11-02 11:55:12 +01:00
$this -> getBackgroundJobsTool () -> enqueue (
BackgroundJobsTool :: PRIO_QUEUE ,
BackgroundJobsTool :: CMD_ADMIN ,
[
'runUpdates' ,
$jobId
],
2021-11-02 16:25:43 +01:00
true ,
$jobId
2019-05-08 15:37:39 +02:00
);
2021-11-02 11:55:12 +01:00
2019-05-08 15:37:39 +02:00
return true ;
}
2019-09-30 16:28:52 +02:00
// See comment above for `isUpdateLocked()`
2019-10-01 13:08:09 +02:00
// prevent continuation of job if worker was already spawned
// (could happens if multiple prio workers are up)
if ( $this -> isUpdateLocked ()) {
2019-09-30 16:28:52 +02:00
$this -> Log -> create ();
$this -> Log -> save ( array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_db_worker' ,
'user_id' => 0 ,
'title' => __ ( 'Issues executing run_updates' ),
'change' => __ ( 'Updates are locked. Stopping worker gracefully' )
2019-09-30 16:28:52 +02:00
));
2019-10-01 13:08:09 +02:00
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'progress' ] = 100 ;
$job [ 'Job' ][ 'message' ] = __ ( 'Update done' );
$this -> Job -> save ( $job );
}
return true ;
2019-09-30 16:28:52 +02:00
}
2019-10-10 12:02:23 +02:00
$this -> changeLockState ( time ());
2020-01-20 13:24:51 +01:00
$this -> __resetUpdateProgress ();
2019-09-30 16:28:52 +02:00
2019-09-30 15:31:49 +02:00
$update_done = 0 ;
2018-07-19 11:48:22 +02:00
foreach ( $updates as $update => $temp ) {
2019-02-10 21:35:23 +01:00
if ( $verbose ) {
echo str_pad ( 'Executing ' . $update , 30 , '.' );
}
2019-09-30 15:31:49 +02:00
if ( ! empty ( $job )) {
2019-10-01 13:08:09 +02:00
$job [ 'Job' ][ 'progress' ] = floor ( $update_done / count ( $updates ) * 100 );
2021-09-26 21:36:22 +02:00
$job [ 'Job' ][ 'message' ] = __ ( 'Running update %s' , $update );
2019-09-30 15:31:49 +02:00
$this -> Job -> save ( $job );
}
2019-10-29 09:57:25 +01:00
$dbUpdateSuccess = $this -> updateMISP ( $update );
2018-07-19 11:48:22 +02:00
if ( $temp ) {
$requiresLogout = true ;
}
2019-10-29 09:57:25 +01:00
if ( $dbUpdateSuccess ) {
2019-08-14 12:13:45 +02:00
$db_version [ 'AdminSetting' ][ 'value' ] = $update ;
$this -> AdminSetting -> save ( $db_version );
2019-10-15 11:44:34 +02:00
$this -> resetUpdateFailNumber ();
2019-10-10 10:14:22 +02:00
} else {
$this -> __increaseUpdateFailNumber ();
2019-08-14 12:13:45 +02:00
}
2019-02-10 21:35:23 +01:00
if ( $verbose ) {
echo " \033 [32mDone \033 [0m " . PHP_EOL ;
}
2019-09-30 15:31:49 +02:00
$update_done ++ ;
}
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'message' ] = __ ( 'Update done' );
2018-07-19 11:48:22 +02:00
}
2019-10-10 12:02:23 +02:00
$this -> changeLockState ( false );
2018-07-19 11:48:22 +02:00
$this -> __queueCleanDB ();
2019-10-01 13:08:09 +02:00
} else {
if ( ! empty ( $job )) {
2021-09-26 21:36:22 +02:00
$job [ 'Job' ][ 'message' ] = __ ( 'Update done in another worker. Gracefully stopping.' );
2019-10-01 13:08:09 +02:00
}
}
// mark current worker as done, as well as queued workers than manages to pass the locks
// (happens if user hit reload before first worker start its job)
if ( ! empty ( $job )) {
$job [ 'Job' ][ 'progress' ] = 100 ;
$this -> Job -> save ( $job );
2018-07-19 11:48:22 +02:00
}
}
if ( $requiresLogout ) {
2020-11-28 15:14:31 +01:00
$this -> refreshSessions ();
2018-07-19 11:48:22 +02:00
}
2019-04-26 09:45:03 +02:00
return true ;
}
2020-11-28 15:14:31 +01:00
/**
* Update date_modified for all users , this will ensure that all users will refresh their session data .
*/
private function refreshSessions ()
{
$this -> User = ClassRegistry :: init ( 'User' );
$this -> User -> updateAll ([ 'date_modified' => time ()]);
}
2019-10-03 13:50:55 +02:00
private function __setUpdateProgress ( $current , $total = false , $toward_db_version = false )
2019-04-26 09:45:03 +02:00
{
$updateProgress = $this -> getUpdateProgress ();
2019-04-29 11:09:04 +02:00
$updateProgress [ 'current' ] = $current ;
2019-04-26 09:45:03 +02:00
if ( $total !== false ) {
2019-04-29 11:09:04 +02:00
$updateProgress [ 'total' ] = $total ;
2019-04-26 09:45:03 +02:00
} else {
$now = new DateTime ();
$updateProgress [ 'time' ][ 'started' ][ $current ] = $now -> format ( 'Y-m-d H:i:s' );
}
2019-10-03 13:50:55 +02:00
if ( $toward_db_version !== false ) {
$updateProgress [ 'toward_db_version' ] = $toward_db_version ;
}
2019-04-26 09:45:03 +02:00
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setPreUpdateTestState ( $state )
{
$updateProgress = $this -> getUpdateProgress ();
$updateProgress [ 'preTestSuccess' ] = $state ;
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setUpdateError ( $index )
{
$updateProgress = $this -> getUpdateProgress ();
$updateProgress [ 'failed_num' ][] = $index ;
$this -> __saveUpdateProgress ( $updateProgress );
}
2020-01-20 13:24:51 +01:00
private function __getEmptyUpdateMessage ()
2019-04-26 09:45:03 +02:00
{
2020-01-20 13:24:51 +01:00
return array (
2019-04-29 11:09:04 +02:00
'commands' => array (),
'results' => array (),
2019-04-26 09:45:03 +02:00
'time' => array ( 'started' => array (), 'elapsed' => array ()),
2019-04-29 11:09:04 +02:00
'current' => '' ,
'total' => '' ,
2019-10-03 13:50:55 +02:00
'failed_num' => array (),
'toward_db_version' => ''
2019-04-26 09:45:03 +02:00
);
2020-01-20 13:24:51 +01:00
}
private function __resetUpdateProgress ()
{
$updateProgress = $this -> __getEmptyUpdateMessage ();
2019-04-26 09:45:03 +02:00
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setUpdateCmdMessages ( $messages )
{
$updateProgress = $this -> getUpdateProgress ();
2019-04-29 11:09:04 +02:00
$updateProgress [ 'commands' ] = $messages ;
2019-04-26 09:45:03 +02:00
$this -> __saveUpdateProgress ( $updateProgress );
}
private function __setUpdateResMessages ( $index , $message )
{
$updateProgress = $this -> getUpdateProgress ();
2019-04-29 11:09:04 +02:00
$updateProgress [ 'results' ][ $index ] = $message ;
2019-04-26 09:45:03 +02:00
$temp = new DateTime ();
$diff = $temp -> diff ( new DateTime ( $updateProgress [ 'time' ][ 'started' ][ $index ]));
$updateProgress [ 'time' ][ 'elapsed' ][ $index ] = $diff -> format ( '%H:%I:%S' );
$this -> __saveUpdateProgress ( $updateProgress );
}
public function getUpdateProgress ()
{
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
$updateProgress = $this -> AdminSetting -> getSetting ( 'update_progress' );
if ( $updateProgress !== false ) {
$updateProgress = json_decode ( $updateProgress , true );
} else {
2020-01-20 13:24:51 +01:00
$updateProgress = $this -> __getEmptyUpdateMessage ();
2019-04-26 09:45:03 +02:00
}
foreach ( $updateProgress as $setting => $value ) {
if ( ! is_array ( $value )) {
2019-10-14 10:49:41 +02:00
if ( is_numeric ( $value )) {
$value = intval ( $value );
}
2019-04-26 09:45:03 +02:00
}
$updateProgress [ $setting ] = $value ;
}
return $updateProgress ;
}
private function __saveUpdateProgress ( $updateProgress )
{
2019-11-27 09:47:25 +01:00
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
2019-04-26 09:45:03 +02:00
$data = json_encode ( $updateProgress );
$this -> AdminSetting -> changeSetting ( 'update_progress' , $data );
}
2019-10-10 12:02:23 +02:00
public function changeLockState ( $locked )
2019-04-26 09:45:03 +02:00
{
2019-11-27 09:47:25 +01:00
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
2019-04-26 09:45:03 +02:00
$this -> AdminSetting -> changeSetting ( 'update_locked' , $locked );
}
private function getUpdateLockState ()
{
if ( ! isset ( $this -> AdminSetting )) {
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
}
$locked = $this -> AdminSetting -> getSetting ( 'update_locked' );
return is_null ( $locked ) ? false : $locked ;
}
2019-10-01 15:22:29 +02:00
public function getLockRemainingTime ()
2019-04-26 09:45:03 +02:00
{
2019-12-09 09:53:18 +01:00
$lockState = $this -> getUpdateLockState ();
2019-04-29 11:09:04 +02:00
if ( $lockState !== false && $lockState !== '' ) {
2019-04-26 09:45:03 +02:00
// if lock is old, still allows the update
// This can be useful if the update process crashes
2019-04-29 11:09:04 +02:00
$diffSec = time () - intval ( $lockState );
2019-04-26 09:45:03 +02:00
if ( Configure :: read ( 'MISP.updateTimeThreshold' )) {
$updateWaitThreshold = intval ( Configure :: read ( 'MISP.updateTimeThreshold' ));
} else {
2019-04-29 11:09:04 +02:00
$this -> Server = ClassRegistry :: init ( 'Server' );
2019-04-26 09:45:03 +02:00
$updateWaitThreshold = intval ( $this -> Server -> serverSettings [ 'MISP' ][ 'updateTimeThreshold' ][ 'value' ]);
}
2019-10-01 15:22:29 +02:00
$remainingTime = $updateWaitThreshold - $diffSec ;
return $remainingTime > 0 ? $remainingTime : 0 ;
} else {
return 0 ;
2019-04-26 09:45:03 +02:00
}
2019-10-01 15:22:29 +02:00
}
public function isUpdateLocked ()
{
$remainingTime = $this -> getLockRemainingTime ();
2019-10-10 12:02:23 +02:00
$failThresholdReached = $this -> UpdateFailNumberReached ();
return $remainingTime > 0 || $failThresholdReached ;
2018-07-19 11:48:22 +02:00
}
2019-10-10 10:14:22 +02:00
public function getUpdateFailNumber ()
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
2019-10-29 09:57:25 +01:00
$updateFailNumber = $this -> AdminSetting -> getSetting ( 'update_fail_number' );
return ( $updateFailNumber !== false && $updateFailNumber !== '' ) ? $updateFailNumber : 0 ;
2019-10-10 10:14:22 +02:00
}
2019-10-15 11:44:34 +02:00
public function resetUpdateFailNumber ()
2019-10-10 10:14:22 +02:00
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$this -> AdminSetting -> changeSetting ( 'update_fail_number' , 0 );
}
2019-10-10 12:02:23 +02:00
public function __increaseUpdateFailNumber ()
2019-10-10 10:14:22 +02:00
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
2019-10-29 09:57:25 +01:00
$updateFailNumber = $this -> AdminSetting -> getSetting ( 'update_fail_number' );
$this -> AdminSetting -> changeSetting ( 'update_fail_number' , $updateFailNumber + 1 );
2019-10-10 10:14:22 +02:00
}
2019-10-10 12:02:23 +02:00
public function UpdateFailNumberReached ()
{
return $this -> getUpdateFailNumber () > 3 ;
}
2018-07-19 11:48:22 +02:00
private function __queueCleanDB ()
{
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$cleanDB = $this -> AdminSetting -> find ( 'first' , array ( 'conditions' => array ( 'setting' => 'clean_db' )));
if ( empty ( $cleanDB )) {
$this -> AdminSetting -> create ();
$cleanDB = array ( 'AdminSetting' => array ( 'setting' => 'clean_db' , 'value' => 1 ));
} else {
$cleanDB [ 'AdminSetting' ][ 'value' ] = 1 ;
}
$this -> AdminSetting -> save ( $cleanDB );
}
private function __runCleanDB ()
{
2021-10-16 09:58:05 +02:00
$cleanDB = $this -> AdminSetting -> getSetting ( 'clean_db' );
2021-10-16 19:32:32 +02:00
if ( $cleanDB === false || $cleanDB == 1 ) {
2018-07-19 11:48:22 +02:00
$this -> cleanCacheFiles ();
2021-10-16 09:58:05 +02:00
$this -> AdminSetting -> changeSetting ( 'clean_db' , 0 );
2018-07-19 11:48:22 +02:00
}
}
2021-11-07 10:40:01 +01:00
/**
* @ param string $db_version
* @ return array
*/
2021-07-27 18:49:40 +02:00
protected function findUpgrades ( $db_version )
2018-07-19 11:48:22 +02:00
{
$updates = array ();
if ( strpos ( $db_version , '.' )) {
$version = explode ( '.' , $db_version );
2021-07-27 18:49:40 +02:00
foreach ( self :: OLD_DB_CHANGES as $major => $rest ) {
2018-07-19 11:48:22 +02:00
if ( $major < $version [ 0 ]) {
continue ;
} elseif ( $major == $version [ 0 ]) {
foreach ( $rest as $minor => $hotfixes ) {
if ( $minor < $version [ 1 ]) {
continue ;
} elseif ( $minor == $version [ 1 ]) {
foreach ( $hotfixes as $hotfix => $requiresLogout ) {
if ( $hotfix > $version [ 2 ]) {
$updates [ $major . '.' . $minor . '.' . $hotfix ] = $requiresLogout ;
}
}
} else {
foreach ( $hotfixes as $hotfix => $requiresLogout ) {
$updates [ $major . '.' . $minor . '.' . $hotfix ] = $requiresLogout ;
}
}
}
}
}
$db_version = 0 ;
}
2021-07-27 18:49:40 +02:00
foreach ( self :: DB_CHANGES as $db_change => $requiresLogout ) {
2018-07-19 11:48:22 +02:00
if ( $db_version < $db_change ) {
$updates [ $db_change ] = $requiresLogout ;
}
}
return $updates ;
}
2020-02-26 14:37:09 +01:00
private function __generateCorrelations ()
{
if ( Configure :: read ( 'MISP.background_jobs' )) {
2021-11-02 16:53:42 +01:00
/** @var Job $job */
$job = ClassRegistry :: init ( 'Job' );
$jobId = $job -> createJob (
2021-11-02 11:55:12 +01:00
'SYSTEM' ,
Job :: WORKER_DEFAULT ,
'generate correlation' ,
'All attributes' ,
'Job created.'
2020-02-26 14:37:09 +01:00
);
2021-11-02 11:55:12 +01:00
$this -> getBackgroundJobsTool () -> enqueue (
BackgroundJobsTool :: DEFAULT_QUEUE ,
BackgroundJobsTool :: CMD_ADMIN ,
[
'jobGenerateCorrelation' ,
$jobId
],
2021-11-02 16:25:43 +01:00
true ,
$jobId
2020-02-26 14:37:09 +01:00
);
2021-11-02 11:55:12 +01:00
2020-02-26 14:37:09 +01:00
}
return true ;
}
2018-07-19 11:48:22 +02:00
public function checkFilename ( $filename )
{
return preg_match ( '@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i' , $filename );
}
2019-09-20 22:15:15 +02:00
/**
* Similar method as `setupRedis` , but this method throw exception if Redis cannot be reached .
* @ return Redis
* @ throws Exception
*/
public function setupRedisWithException ()
2018-07-19 11:48:22 +02:00
{
2021-02-15 18:28:41 +01:00
if ( self :: $__redisConnection ) {
return self :: $__redisConnection ;
2019-08-16 19:26:12 +02:00
}
if ( ! class_exists ( 'Redis' )) {
2022-03-19 12:31:55 +01:00
throw new Exception ( " Class Redis doesn't exists. Please install redis extension for PHP. " );
2018-07-19 11:48:22 +02:00
}
2019-08-16 19:26:12 +02:00
$host = Configure :: read ( 'MISP.redis_host' ) ? : '127.0.0.1' ;
$port = Configure :: read ( 'MISP.redis_port' ) ? : 6379 ;
$database = Configure :: read ( 'MISP.redis_database' ) ? : 13 ;
2018-07-19 11:48:22 +02:00
$pass = Configure :: read ( 'MISP.redis_password' );
2019-08-16 19:26:12 +02:00
$redis = new Redis ();
2022-05-06 20:56:40 +02:00
if ( ! $redis -> connect ( $host , ( int ) $port )) {
2019-09-20 22:15:15 +02:00
throw new Exception ( " Could not connect to Redis: { $redis -> getLastError () } " );
2018-07-19 11:48:22 +02:00
}
if ( ! empty ( $pass )) {
2019-09-20 22:15:15 +02:00
if ( ! $redis -> auth ( $pass )) {
throw new Exception ( " Could not authenticate to Redis: { $redis -> getLastError () } " );
}
2018-07-19 11:48:22 +02:00
}
2019-09-20 22:15:15 +02:00
if ( ! $redis -> select ( $database )) {
throw new Exception ( " Could not select Redis database $database : { $redis -> getLastError () } " );
2018-07-19 11:48:22 +02:00
}
2019-09-20 22:15:15 +02:00
2021-02-15 18:28:41 +01:00
self :: $__redisConnection = $redis ;
2018-07-19 11:48:22 +02:00
return $redis ;
}
2019-09-20 22:15:15 +02:00
/**
* Method for backward compatibility .
* @ deprecated
* @ see AppModel :: setupRedisWithException
* @ return bool | Redis
*/
public function setupRedis ()
{
try {
return $this -> setupRedisWithException ();
} catch ( Exception $e ) {
return false ;
}
}
2019-03-05 12:24:56 +01:00
public function getKafkaPubTool ()
{
if ( ! $this -> loadedKafkaPubTool ) {
2021-11-07 10:40:01 +01:00
App :: uses ( 'KafkaPubTool' , 'Tools' );
$kafkaPubTool = new KafkaPubTool ();
$rdkafkaIni = Configure :: read ( 'Plugin.Kafka_rdkafka_config' );
2022-04-17 18:25:51 +02:00
$rdkafkaIni = mb_ereg_replace ( " / \ : \ / \ // " , '' , $rdkafkaIni );
2021-11-07 10:40:01 +01:00
$kafkaConf = array ();
if ( ! empty ( $rdkafkaIni )) {
$kafkaConf = parse_ini_file ( $rdkafkaIni );
}
$brokers = Configure :: read ( 'Plugin.Kafka_brokers' );
$kafkaPubTool -> initTool ( $brokers , $kafkaConf );
$this -> loadedKafkaPubTool = $kafkaPubTool ;
2019-03-05 12:24:56 +01:00
}
return $this -> loadedKafkaPubTool ;
}
2021-10-10 18:41:44 +02:00
public function publishKafkaNotification ( $topicName , $data , $action = false )
{
$kafkaTopic = $this -> kafkaTopic ( $topicName );
if ( $kafkaTopic ) {
2019-03-05 12:24:56 +01:00
$this -> getKafkaPubTool () -> publishJson ( $kafkaTopic , $data , $action );
}
}
2021-07-25 20:03:32 +02:00
/**
* @ return PubSubTool
*/
2018-07-19 11:48:22 +02:00
public function getPubSubTool ()
{
2021-07-25 20:03:32 +02:00
if ( ! self :: $loadedPubSubTool ) {
2020-05-07 15:50:54 +02:00
App :: uses ( 'PubSubTool' , 'Tools' );
$pubSubTool = new PubSubTool ();
$pubSubTool -> initTool ();
2021-07-25 20:03:32 +02:00
self :: $loadedPubSubTool = $pubSubTool ;
2018-07-19 11:48:22 +02:00
}
2021-07-25 20:03:32 +02:00
return self :: $loadedPubSubTool ;
2018-07-19 11:48:22 +02:00
}
2021-10-31 10:46:28 +01:00
protected function getElasticSearchTool ()
2018-07-19 11:48:22 +02:00
{
if ( ! $this -> elasticSearchClient ) {
2021-10-31 10:46:28 +01:00
App :: uses ( 'ElasticSearchClient' , 'Tools' );
$client = new ElasticSearchClient ();
$client -> initTool ();
$this -> elasticSearchClient = $client ;
2018-07-19 11:48:22 +02:00
}
return $this -> elasticSearchClient ;
}
2021-10-21 13:39:26 +02:00
/**
* @ return BackgroundJobsTool
*/
2021-10-27 11:33:32 +02:00
public function getBackgroundJobsTool () : BackgroundJobsTool
2021-10-21 13:39:26 +02:00
{
if ( ! self :: $loadedBackgroundJobsTool ) {
App :: uses ( 'BackgroundJobsTool' , 'Tools' );
2021-10-27 11:33:32 +02:00
// TODO: remove after CakeResque is deprecated
2021-11-04 11:39:24 +01:00
$settings = [ 'enabled' => false ];
2021-11-12 09:43:29 +01:00
if ( Configure :: read ( 'SimpleBackgroundJobs.enabled' )) {
2021-11-05 15:46:42 +01:00
$settings = Configure :: read ( 'SimpleBackgroundJobs' );
2021-10-27 11:33:32 +02:00
}
2021-11-09 08:43:39 +01:00
$backgroundJobsTool = new BackgroundJobsTool ( $settings );
self :: $loadedBackgroundJobsTool = $backgroundJobsTool ;
2021-10-21 13:39:26 +02:00
}
2021-11-09 08:43:39 +01:00
return self :: $loadedBackgroundJobsTool ;
2021-10-21 13:39:26 +02:00
}
2018-07-19 11:48:22 +02:00
// generate a generic subquery - options needs to include conditions
2021-10-30 19:13:17 +02:00
protected function subQueryGenerator ( AppModel $model , array $options , $lookupKey , $negation = false )
2018-07-19 11:48:22 +02:00
{
$defaults = array (
'fields' => array ( '*' ),
2018-08-02 15:40:53 +02:00
'table' => $model -> table ,
2018-07-19 11:48:22 +02:00
'alias' => $model -> alias ,
'limit' => null ,
'offset' => null ,
'joins' => array (),
'conditions' => array (),
2020-06-14 20:23:48 +02:00
'group' => false ,
'recursive' => - 1
2018-07-19 11:48:22 +02:00
);
$params = array ();
2021-10-30 19:13:17 +02:00
foreach ( $defaults as $key => $defaultValue ) {
2018-08-02 15:40:53 +02:00
if ( isset ( $options [ $key ])) {
$params [ $key ] = $options [ $key ];
2018-07-19 11:48:22 +02:00
} else {
2021-10-30 19:13:17 +02:00
$params [ $key ] = $defaultValue ;
2018-07-19 11:48:22 +02:00
}
}
2021-10-30 19:13:17 +02:00
$db = $model -> getDataSource ();
$subQuery = $db -> buildStatement ( $params , $model );
2018-08-02 15:40:53 +02:00
if ( $negation ) {
$subQuery = $lookupKey . ' NOT IN (' . $subQuery . ') ' ;
} else {
$subQuery = $lookupKey . ' IN (' . $subQuery . ') ' ;
}
2018-07-19 11:48:22 +02:00
$conditions = array (
$db -> expression ( $subQuery ) -> value
);
return $conditions ;
}
// start a benchmark run for the given bench name
public function benchmarkInit ( $name = 'default' )
{
$this -> __profiler [ $name ][ 'start' ] = microtime ( true );
if ( empty ( $this -> __profiler [ $name ][ 'memory_start' ])) {
$this -> __profiler [ $name ][ 'memory_start' ] = memory_get_usage ();
}
return true ;
}
// calculate the duration from the init time to the current point in execution. Aggregate flagged executions will increment the duration instead of just setting it
public function benchmark ( $name = 'default' , $aggregate = false , $memory_chart = false )
{
if ( ! empty ( $this -> __profiler [ $name ][ 'start' ])) {
if ( $aggregate ) {
if ( ! isset ( $this -> __profiler [ $name ][ 'duration' ])) {
$this -> __profiler [ $name ][ 'duration' ] = 0 ;
}
if ( ! isset ( $this -> __profiler [ $name ][ 'executions' ])) {
$this -> __profiler [ $name ][ 'executions' ] = 0 ;
}
$this -> __profiler [ $name ][ 'duration' ] += microtime ( true ) - $this -> __profiler [ $name ][ 'start' ];
$this -> __profiler [ $name ][ 'executions' ] ++ ;
$currentUsage = memory_get_usage ();
if ( $memory_chart ) {
$this -> __profiler [ $name ][ 'memory_chart' ][] = $currentUsage - $this -> __profiler [ $name ][ 'memory_start' ];
}
if (
empty ( $this -> __profiler [ $name ][ 'memory_peak' ]) ||
$this -> __profiler [ $name ][ 'memory_peak' ] < ( $currentUsage - $this -> __profiler [ $name ][ 'memory_start' ])
) {
$this -> __profiler [ $name ][ 'memory_peak' ] = $currentUsage - $this -> __profiler [ $name ][ 'memory_start' ];
}
} else {
$this -> __profiler [ $name ][ 'memory_peak' ] = memory_get_usage () - $this -> __profiler [ $name ][ 'memory_start' ];
$this -> __profiler [ $name ][ 'duration' ] = microtime ( true ) - $this -> __profiler [ $name ][ 'start' ];
}
}
return true ;
}
// return the results of the benchmark(s). If no name is set all benchmark results are returned in an array.
public function benchmarkResult ( $name = false )
{
if ( $name ) {
return array ( $name => $this -> __profiler [ $name ][ 'duration' ]);
} else {
$results = array ();
foreach ( $this -> __profiler as $name => $benchmark ) {
if ( ! empty ( $benchmark [ 'duration' ])) {
$results [ $name ] = $benchmark ;
unset ( $results [ $name ][ 'start' ]);
unset ( $results [ $name ][ 'memory_start' ]);
}
}
return $results ;
}
}
public function benchmarkCustomAdd ( $valueToAdd = 0 , $name = 'default' , $customName = 'custom' )
{
if ( empty ( $this -> __profiler [ $name ][ 'custom' ][ $customName ])) {
$this -> __profiler [ $name ][ 'custom' ][ $customName ] = 0 ;
}
$this -> __profiler [ $name ][ 'custom' ][ $customName ] += $valueToAdd ;
}
private function __forceSettings ()
{
$settingsToForce = array (
'Session.autoRegenerate' => false ,
'Session.checkAgent' => false
);
$server = ClassRegistry :: init ( 'Server' );
foreach ( $settingsToForce as $setting => $value ) {
$server -> serverSettingsSaveValue ( $setting , $value );
}
return true ;
}
2018-08-03 11:52:20 +02:00
2020-03-02 23:09:47 +01:00
public function setupHttpSocket ( $server , $HttpSocket = null , $timeout = false )
2018-08-03 11:52:20 +02:00
{
if ( empty ( $HttpSocket )) {
App :: uses ( 'SyncTool' , 'Tools' );
$syncTool = new SyncTool ();
2020-03-02 23:09:47 +01:00
$HttpSocket = $syncTool -> setupHttpSocket ( $server , $timeout );
2018-08-03 11:52:20 +02:00
}
return $HttpSocket ;
}
2020-09-26 19:53:52 +02:00
/**
* @ param array $server
2021-03-04 18:29:26 +01:00
* @ param string $model
2020-09-26 19:53:52 +02:00
* @ return array []
* @ throws JsonException
*/
2022-04-18 17:57:38 +02:00
public function setupSyncRequest ( array $server , $model = 'Server' )
2018-08-03 11:52:20 +02:00
{
2020-10-06 17:27:53 +02:00
$version = implode ( '.' , $this -> checkMISPVersion ());
2021-03-04 18:29:26 +01:00
$commit = $this -> checkMIPSCommit ();
2021-11-07 16:58:42 +01:00
$authkey = $server [ $model ][ 'authkey' ];
App :: uses ( 'EncryptedValue' , 'Tools' );
if ( EncryptedValue :: isEncrypted ( $authkey )) {
$authkey = ( string ) new EncryptedValue ( $authkey );
}
return array (
2020-09-26 19:53:52 +02:00
'header' => array (
2021-11-07 16:58:42 +01:00
'Authorization' => $authkey ,
2020-09-26 19:53:52 +02:00
'Accept' => 'application/json' ,
'Content-Type' => 'application/json' ,
2021-03-04 18:29:26 +01:00
'User-Agent' => 'MISP ' . $version . ( empty ( $commit ) ? '' : ' - #' . $commit ),
2020-09-26 19:53:52 +02:00
)
2018-08-03 11:52:20 +02:00
);
}
2018-08-03 13:22:17 +02:00
2020-09-26 19:53:52 +02:00
/**
* Returns MISP version from VERSION . json file as array with major , minor and hotfix keys .
*
* @ return array
* @ throws JsonException
*/
public function checkMISPVersion ()
2018-08-03 13:22:17 +02:00
{
2020-09-26 19:53:52 +02:00
static $versionArray ;
if ( $versionArray === null ) {
2021-11-07 10:40:01 +01:00
$content = FileAccessTool :: readFromFile ( ROOT . DS . 'VERSION.json' );
$versionArray = JsonTool :: decode ( $content );
2018-08-03 13:22:17 +02:00
}
2020-09-26 19:53:52 +02:00
return $versionArray ;
}
/**
* Returns MISP commit hash .
*
* @ return false | string
*/
protected function checkMIPSCommit ()
{
static $commit ;
if ( $commit === null ) {
2021-11-14 11:30:17 +01:00
App :: uses ( 'GitTool' , 'Tools' );
try {
$commit = GitTool :: currentCommit ();
} catch ( Exception $e ) {
$this -> logException ( 'Could not get current git commit' , $e , LOG_NOTICE );
2020-09-26 19:53:52 +02:00
$commit = false ;
}
2018-08-03 13:22:17 +02:00
}
2020-09-26 19:53:52 +02:00
return $commit ;
2018-08-03 13:22:17 +02:00
}
2018-08-09 07:44:53 +02:00
// take filters in the {"OR" => [foo], "NOT" => [bar]} format along with conditions and set the conditions
2018-10-08 10:17:57 +02:00
public function generic_add_filter ( $conditions , & $filter , $keys )
2018-08-09 07:44:53 +02:00
{
$operator_composition = array (
'NOT' => 'AND' ,
'OR' => 'OR' ,
'AND' => 'AND'
);
if ( ! is_array ( $keys )) {
$keys = array ( $keys );
}
2018-09-03 17:54:37 +02:00
if ( ! isset ( $filter [ 'OR' ]) && ! isset ( $filter [ 'AND' ]) && ! isset ( $filter [ 'NOT' ])) {
2018-08-09 07:44:53 +02:00
return $conditions ;
}
foreach ( $filter as $operator => $filters ) {
$temp = array ();
2018-11-23 14:11:33 +01:00
if ( ! is_array ( $filters )) {
$filters = array ( $filters );
}
2018-08-09 07:44:53 +02:00
foreach ( $filters as $f ) {
2018-11-23 14:11:33 +01:00
if ( $f === - 1 ) {
foreach ( $keys as $key ) {
$temp [ 'OR' ][ $key ][] = - 1 ;
}
continue ;
}
2018-08-09 07:44:53 +02:00
// split the filter params into two lists, one for substring searches one for exact ones
2018-10-11 20:20:27 +02:00
if ( is_string ( $f ) && ( $f [ strlen ( $f ) - 1 ] === '%' || $f [ 0 ] === '%' )) {
2018-08-09 07:44:53 +02:00
foreach ( $keys as $key ) {
if ( $operator === 'NOT' ) {
$temp [] = array ( $key . ' NOT LIKE' => $f );
} else {
$temp [] = array ( $key . ' LIKE' => $f );
}
}
} else {
foreach ( $keys as $key ) {
if ( $operator === 'NOT' ) {
$temp [ $key . ' !=' ][] = $f ;
} else {
2018-08-09 15:11:57 +02:00
$temp [ 'OR' ][ $key ][] = $f ;
2018-08-09 07:44:53 +02:00
}
}
}
}
2018-11-23 14:11:33 +01:00
$conditions [ 'AND' ][] = array ( $operator_composition [ $operator ] => $temp );
2018-08-09 07:44:53 +02:00
if ( $operator !== 'NOT' ) {
unset ( $filter [ $operator ]);
}
}
return $conditions ;
}
/*
* Get filters in one of the following formats :
* [ foo , bar ]
* [ " OR " => [ foo , bar ], " NOT " => [ baz ]]
* " foo "
* " foo&&bar&&!baz "
* and convert it into the same format [ " OR " => [ foo , bar ], " NOT " => [ baz ]]
*/
public function convert_filters ( $filter )
{
if ( ! is_array ( $filter )) {
$temp = explode ( '&&' , $filter );
$filter = array ();
foreach ( $temp as $f ) {
2020-11-05 10:13:05 +01:00
$f = strval ( $f );
2018-08-09 07:44:53 +02:00
if ( $f [ 0 ] === '!' ) {
2019-04-23 14:02:27 +02:00
$filter [ 'NOT' ][] = substr ( $f , 1 );
2018-08-09 07:44:53 +02:00
} else {
$filter [ 'OR' ][] = $f ;
}
}
return $filter ;
}
if ( ! isset ( $filter [ 'OR' ]) && ! isset ( $filter [ 'NOT' ]) && ! isset ( $filter [ 'AND' ])) {
$temp = array ();
foreach ( $filter as $param ) {
2020-11-05 10:13:05 +01:00
$param = strval ( $param );
2018-11-23 14:11:33 +01:00
if ( ! empty ( $param )) {
if ( $param [ 0 ] === '!' ) {
$temp [ 'NOT' ][] = substr ( $param , 1 );
} else {
$temp [ 'OR' ][] = $param ;
}
}
2018-08-09 07:44:53 +02:00
}
$filter = $temp ;
}
return $filter ;
}
2018-09-28 09:21:29 +02:00
2018-11-23 14:11:33 +01:00
public function convert_to_memory_limit_to_mb ( $val )
{
$val = trim ( $val );
if ( $val == - 1 ) {
// default to 8GB if no limit is set
return 8 * 1024 ;
}
$unit = $val [ strlen ( $val ) - 1 ];
if ( is_numeric ( $unit )) {
$unit = 'b' ;
} else {
$val = intval ( $val );
}
$unit = strtolower ( $unit );
switch ( $unit ) {
case 'g' :
$val *= 1024 ;
// no break
case 'm' :
$val *= 1024 ;
// no break
case 'k' :
$val *= 1024 ;
}
return $val / ( 1024 * 1024 );
}
private function __bumpReferences ()
{
$this -> Event = ClassRegistry :: init ( 'Event' );
$this -> AdminSetting = ClassRegistry :: init ( 'AdminSetting' );
$existingSetting = $this -> AdminSetting -> find ( 'first' , array (
'conditions' => array ( 'AdminSetting.setting' => 'update_23' )
));
if ( empty ( $existingSetting )) {
$this -> AdminSetting -> create ();
$data = array (
'setting' => 'update_23' ,
'value' => 1
);
$this -> AdminSetting -> save ( $data );
$references = $this -> Event -> Object -> ObjectReference -> find ( 'list' , array (
'recursive' => - 1 ,
'fields' => array ( 'ObjectReference.event_id' , 'ObjectReference.event_id' ),
'group' => array ( 'ObjectReference.event_id' )
));
$event_ids = array ();
$object_ids = array ();
foreach ( $references as $reference ) {
$event = $this -> Event -> find ( 'first' , array (
'conditions' => array (
'Event.id' => $reference ,
'Event.locked' => 0
),
'recursive' => - 1 ,
'fields' => array ( 'Event.id' , 'Event.locked' )
));
if ( ! empty ( $event )) {
$event_ids [] = $event [ 'Event' ][ 'id' ];
$event_references = $this -> Event -> Object -> ObjectReference -> find ( 'list' , array (
'conditions' => array ( 'ObjectReference.event_id' => $reference ),
'recursive' => - 1 ,
'fields' => array ( 'ObjectReference.object_id' , 'ObjectReference.object_id' )
));
$object_ids = array_merge ( $object_ids , array_values ( $event_references ));
}
}
if ( ! empty ( $object_ids )) {
$this -> Event -> Object -> updateAll (
array (
'Object.timestamp' => 'Object.timestamp + 1'
),
array ( 'Object.id' => $object_ids )
);
$this -> Event -> updateAll (
array (
'Event.timestamp' => 'Event.timestamp + 1'
),
array ( 'Event.id' => $event_ids )
);
}
$this -> Log = ClassRegistry :: init ( 'Log' );
$this -> Log -> create ();
$entry = array (
2021-11-07 10:40:01 +01:00
'org' => 'SYSTEM' ,
'model' => 'Server' ,
'model_id' => 0 ,
'email' => 'SYSTEM' ,
'action' => 'update_database' ,
'user_id' => 0 ,
'title' => 'Bumped the timestamps of locked events containing object references.' ,
'change' => sprintf ( 'Event timestamps updated: %s; Object timestamps updated: %s' , count ( $event_ids ), count ( $object_ids ))
2018-11-23 14:11:33 +01:00
);
$this -> Log -> save ( $entry );
}
return true ;
}
2018-11-26 09:12:01 +01:00
2019-02-10 13:08:12 +01:00
public function generateRandomFileName ()
{
return ( new RandomTool ()) -> random_str ( false , 12 );
}
2019-02-06 17:46:36 +01:00
public function resolveTimeDelta ( $delta )
{
if ( is_numeric ( $delta )) {
return $delta ;
}
$multiplierArray = array ( 'd' => 86400 , 'h' => 3600 , 'm' => 60 , 's' => 1 );
$lastChar = strtolower ( substr ( $delta , - 1 ));
2021-09-13 23:58:10 +02:00
if ( ! is_numeric ( $lastChar ) && isset ( $multiplierArray [ $lastChar ])) {
2019-02-06 17:46:36 +01:00
$multiplier = $multiplierArray [ $lastChar ];
$delta = substr ( $delta , 0 , - 1 );
2021-09-13 23:58:10 +02:00
} else if ( strtotime ( $delta ) !== false ) {
2019-02-06 17:46:36 +01:00
return strtotime ( $delta );
} else {
// invalid filter, make sure we don't return anything
return time () + 1 ;
}
if ( ! is_numeric ( $delta )) {
// Same here. (returning false dumps the whole database)
return time () + 1 ;
}
return time () - ( $delta * $multiplier );
}
2019-03-25 17:35:02 +01:00
private function __fixServerPullPushRules ()
{
$this -> Server = ClassRegistry :: init ( 'Server' );
$servers = $this -> Server -> find ( 'all' , array ( 'recursive' => - 1 ));
foreach ( $servers as $server ) {
$changed = false ;
if ( empty ( $server [ 'Server' ][ 'pull_rules' ])) {
$server [ 'Server' ][ 'pull_rules' ] = '[]' ;
$changed = true ;
}
if ( empty ( $server [ 'Server' ][ 'push_rules' ])) {
$server [ 'Server' ][ 'push_rules' ] = '[]' ;
$changed = true ;
}
if ( $changed ) {
$this -> Server -> save ( $server );
}
}
}
2019-10-08 11:43:56 +02:00
2021-03-07 11:12:03 +01:00
/**
* Optimised version of CakePHP _findList method when just one or two fields are set from same model
* @ param string $state
* @ param array $query
* @ param array $results
* @ return array
*/
protected function _findList ( $state , $query , $results = [])
{
if ( $state === 'before' ) {
return parent :: _findList ( $state , $query , $results );
}
if ( empty ( $results )) {
return [];
}
if ( $query [ 'list' ][ 'groupPath' ] === null ) {
$keyPath = explode ( '.' , $query [ 'list' ][ 'keyPath' ]);
$valuePath = explode ( '.' , $query [ 'list' ][ 'valuePath' ]);
if ( $keyPath [ 1 ] === $valuePath [ 1 ]) { // same model
return array_column ( array_column ( $results , $keyPath [ 1 ]), $valuePath [ 2 ], $keyPath [ 2 ]);
}
}
return parent :: _findList ( $state , $query , $results );
}
2020-12-22 23:43:17 +01:00
/**
* Find method that allows to fetch just one column from database .
* @ param $state
* @ param $query
* @ param array $results
2020-12-31 22:48:27 +01:00
* @ return array
2021-03-04 19:08:54 +01:00
* @ throws InvalidArgumentException
2020-12-22 23:43:17 +01:00
*/
protected function _findColumn ( $state , $query , $results = array ())
{
if ( $state === 'before' ) {
2021-08-30 17:51:17 +02:00
if ( isset ( $query [ 'fields' ]) && is_array ( $query [ 'fields' ]) && count ( $query [ 'fields' ]) === 1 ) {
2020-12-22 23:43:17 +01:00
if ( strpos ( $query [ 'fields' ][ 0 ], '.' ) === false ) {
$query [ 'fields' ][ 0 ] = $this -> alias . '.' . $query [ 'fields' ][ 0 ];
}
$query [ 'column' ] = $query [ 'fields' ][ 0 ];
if ( isset ( $query [ 'unique' ]) && $query [ 'unique' ]) {
$query [ 'fields' ] = array ( " DISTINCT { $query [ 'fields' ][ 0 ] } " );
} else {
$query [ 'fields' ] = array ( $query [ 'fields' ][ 0 ]);
}
2021-03-04 19:08:54 +01:00
} else if ( ! isset ( $query [ 'fields' ])) {
throw new InvalidArgumentException ( " This method requires `fields` option defined. " );
2020-12-22 23:43:17 +01:00
} else {
2021-03-04 19:08:54 +01:00
throw new InvalidArgumentException ( " Invalid number of column, expected one, " . count ( $query [ 'fields' ]) . " given " );
2020-12-22 23:43:17 +01:00
}
if ( ! isset ( $query [ 'recursive' ])) {
$query [ 'recursive' ] = - 1 ;
}
return $query ;
}
// Faster version of `Hash::extract`
foreach ( explode ( '.' , $query [ 'column' ]) as $part ) {
$results = array_column ( $results , $part );
}
return $results ;
}
/**
* @ param string $field
* @ param AppModel $model
* @ param array $conditions
*/
2020-12-11 22:01:47 +01:00
public function addCountField ( $field , AppModel $model , array $conditions )
{
$db = $this -> getDataSource ();
$subQuery = $db -> buildStatement (
array (
'fields' => [ 'COUNT(*)' ],
'table' => $db -> fullTableName ( $model ),
'alias' => $model -> alias ,
'conditions' => $conditions ,
),
$model
);
$this -> virtualFields [ $field ] = $subQuery ;
}
2019-09-24 20:52:36 +02:00
/**
2020-04-26 21:44:22 +02:00
* Log exception with backtrace and with nested exceptions .
*
2019-09-24 20:52:36 +02:00
* @ param string $message
* @ param Exception $exception
* @ param int $type
* @ return bool
*/
protected function logException ( $message , Exception $exception , $type = LOG_ERR )
{
2020-01-26 18:17:49 +01:00
$message .= " \n " ;
do {
2021-11-07 10:40:01 +01:00
$message .= sprintf ( " [%s] %s " , get_class ( $exception ), $exception -> getMessage ());
2020-01-26 18:17:49 +01:00
$message .= " \n Stack Trace: \n " . $exception -> getTraceAsString ();
$exception = $exception -> getPrevious ();
} while ( $exception !== null );
2019-09-24 20:52:36 +02:00
return $this -> log ( $message , $type );
}
2020-04-28 15:31:27 +02:00
2020-06-19 10:22:19 +02:00
/**
* Decodes JSON string and throws exception if string is not valid JSON or if is not array .
*
* @ param string $json
* @ return array
* @ throws JsonException
* @ throws UnexpectedValueException
*/
2020-07-27 12:10:53 +02:00
public function jsonDecode ( $json )
2020-06-19 10:22:19 +02:00
{
2022-04-10 09:38:25 +02:00
$decoded = JsonTool :: decode ( $json );
2020-06-19 10:22:19 +02:00
if ( ! is_array ( $decoded )) {
throw new UnexpectedValueException ( 'JSON must be array type, get ' . gettype ( $decoded ));
}
return $decoded ;
}
2020-07-30 09:17:27 +02:00
2021-10-05 19:10:55 +02:00
/**
* Faster version of default `hasAny` method
* @ param array | null $conditions
* @ return bool
*/
public function hasAny ( $conditions = null )
{
2022-05-29 12:08:01 +02:00
return ( bool ) $this -> find ( 'first' , [
2021-10-05 19:10:55 +02:00
'fields' => [ $this -> alias . '.' . $this -> primaryKey ],
'conditions' => $conditions ,
'recursive' => - 1 ,
'callbacks' => false ,
2021-10-23 19:54:38 +02:00
'order' => [], // disable order
2022-05-29 12:08:01 +02:00
]);
}
/**
* Faster version of original `isUnique` method
* { @ inheritDoc }
*/
public function isUnique ( $fields , $or = true )
{
if ( is_array ( $or )) {
$isRule = (
array_key_exists ( 'rule' , $or ) &&
array_key_exists ( 'required' , $or ) &&
array_key_exists ( 'message' , $or )
);
if ( ! $isRule ) {
$args = func_get_args ();
$fields = $args [ 1 ];
$or = isset ( $args [ 2 ]) ? $args [ 2 ] : true ;
}
}
if ( ! is_array ( $fields )) {
$fields = func_get_args ();
$fieldCount = count ( $fields ) - 1 ;
if ( is_bool ( $fields [ $fieldCount ])) {
$or = $fields [ $fieldCount ];
unset ( $fields [ $fieldCount ]);
}
}
foreach ( $fields as $field => $value ) {
if ( is_numeric ( $field )) {
unset ( $fields [ $field ]);
$field = $value ;
$value = null ;
if ( isset ( $this -> data [ $this -> alias ][ $field ])) {
$value = $this -> data [ $this -> alias ][ $field ];
}
}
if ( strpos ( $field , '.' ) === false ) {
unset ( $fields [ $field ]);
$fields [ $this -> alias . '.' . $field ] = $value ;
}
}
if ( $or ) {
$fields = array ( 'or' => $fields );
}
if ( ! empty ( $this -> id )) {
$fields [ $this -> alias . '.' . $this -> primaryKey . ' !=' ] = $this -> id ;
}
return ! $this -> hasAny ( $fields );
}
/**
* Faster version of original `exists` method
* { @ inheritDoc }
*/
public function exists ( $id = null )
{
if ( $id === null ) {
$id = $this -> getID ();
}
if ( $id === false || $this -> useTable === false ) {
return false ;
}
return $this -> hasAny ([ $this -> alias . '.' . $this -> primaryKey => $id ]);
2021-10-05 19:10:55 +02:00
}
2021-10-28 13:19:20 +02:00
/**
* @ param int $value Timestamp in microseconds
* @ return string
*/
protected function microTimestampToIso ( $value )
{
$sec = ( int )( $value / 1000000 );
$micro = $value % 1000000 ;
$micro = str_pad ( $micro , 6 , " 0 " , STR_PAD_LEFT );
return DateTime :: createFromFormat ( 'U.u' , " $sec . $micro " ) -> format ( 'Y-m-d\TH:i:s.uP' );
}
2020-08-13 15:58:42 +02:00
/**
* @ return AttachmentTool
*/
protected function loadAttachmentTool ()
{
if ( $this -> attachmentTool === null ) {
$this -> attachmentTool = new AttachmentTool ();
}
return $this -> attachmentTool ;
}
2020-10-16 17:34:55 +02:00
2020-10-08 19:27:49 +02:00
/**
* @ return AttachmentScan
*/
protected function loadAttachmentScan ()
{
if ( $this -> AttachmentScan === null ) {
$this -> AttachmentScan = ClassRegistry :: init ( 'AttachmentScan' );
}
return $this -> AttachmentScan ;
}
2020-10-16 17:34:55 +02:00
/**
* @ return Log
*/
protected function loadLog ()
{
if ( ! isset ( $this -> Log )) {
$this -> Log = ClassRegistry :: init ( 'Log' );
}
return $this -> Log ;
}
2021-10-10 18:41:44 +02:00
/**
* @ param string $name
* @ return string | null Null when Kafka is not enabled , topic is not enabled or topic is not defined
*/
protected function kafkaTopic ( $name )
{
static $kafkaEnabled ;
if ( $kafkaEnabled === null ) {
$kafkaEnabled = ( bool ) Configure :: read ( 'Plugin.Kafka_enable' );
}
if ( $kafkaEnabled ) {
if ( ! Configure :: read ( " Plugin.Kafka_ { $name } _notifications_enable " )) {
return null ;
}
return Configure :: read ( " Plugin.Kafka_ { $name } _notifications_topic " ) ? : null ;
}
return null ;
}
2021-10-17 16:17:17 +02:00
/**
* @ param string $name
* @ return bool
*/
protected function pubToZmq ( $name )
{
static $zmqEnabled ;
if ( $zmqEnabled === null ) {
$zmqEnabled = ( bool ) Configure :: read ( 'Plugin.ZeroMQ_enable' );
}
if ( $zmqEnabled ) {
return Configure :: read ( " Plugin.ZeroMQ_ { $name } _notifications_enable " );
}
return false ;
}
2022-05-14 10:17:06 +02:00
/**
* @ return bool Returns true if database is MySQL / Mariadb , false for PostgreSQL
*/
protected function isMysql ()
{
$dataSource = ConnectionManager :: getDataSource ( 'default' );
$dataSourceName = $dataSource -> config [ 'datasource' ];
2022-05-19 10:58:25 +02:00
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql ;
2022-05-14 10:17:06 +02:00
}
2012-03-15 15:06:45 +01:00
}