fix: Fix and cleanup script for a specific bug

- rare occurance, but some MISP servers enter an upgrade loop causing massive amounts of log entries
- this patch cleans up the bug preventing further upgrade loops as well as offers a script to clean up the fallout

app/Console/Command/AdminShell.php

@@ -2,7 +2,7 @@
 App::uses('AppShell', 'Console/Command');
 class AdminShell extends AppShell
 {
-	public $uses = array('Event');
+	public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation');
 
 	public function jobGenerateCorrelation() {
 		$jobId = $this->args[0];
@@ -45,4 +45,17 @@ class AdminShell extends AppShell
 		$this->Job->saveField('message', 'Job done.');
 		$this->Job->saveField('status', 4);
 	}
+
+	public function prune_update_logs() {
+		$jobId = $this->args[0];
+		$user_id = $this->args[1];
+		$user = $this->User->getAuthUser($user_id);
+		$this->loadModel('Job');
+		$this->Job->id = $jobId;
+		$this->loadModel('Log');
+		$this->Log->pruneUpdateLogs($jobId, $user);
+		$this->Job->saveField('progress', 100);
+		$this->Job->saveField('message', 'Job done.');
+		$this->Job->saveField('status', 4);
+	}
 }

app/Controller/AttributesController.php

@@ -238,7 +238,7 @@ class AttributesController extends AppController {
 							$message .= '[' . $k . ']: ' . $v[0] . PHP_EOL;
 						}
 						throw new NotFoundException('Could not save the attribute. ' . $message);
-					}  else if ($this->request->is('ajax')) {
+					} else if ($this->request->is('ajax')) {
 						$this->autoRender = false;
 						return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $this->Attribute->validationErrors)),'status'=>200));
 					} else {

app/Controller/LogsController.php

@@ -287,4 +287,17 @@ class LogsController extends AppController {
 		$this->set('data', $data);
 		$this->set('_serialize', 'data');
 	}
+
+	public function pruneUpdateLogs() {
+		if (!$this->request->is('post')) {
+			//throw new MethodNotAllowedException('This functionality is only accessible via POST requests');
+		}
+		$this->Log->pruneUpdateLogsRouter($this->Auth->user());
+		if (Configure::read('MISP.background_jobs')) {
+			$this->Session->setFlash('The pruning job is queued.');
+		} else {
+			$this->Session->setFlash('The pruning is complete.');
+		}
+		$this->redirect($this->referer());
+	}
 }

app/Model/AppModel.php

@@ -772,7 +772,16 @@ class AppModel extends Model {
 			$requiresLogout = true;
 		} else {
 			$this->__runCleanDB();
-			$db_version = $this->AdminSetting->find('first', array('conditions' => array('setting' => 'db_version')));
+			$db_version = $this->AdminSetting->find('all', array('conditions' => array('setting' => 'db_version')));
+			if (count($db_version) > 1) {
+				// we ran into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
+				foreach ($db_version as $k => $v) {
+					if ($k > 0) {
+						$this->AdminSetting->delete($v['AdminSetting']['id']);
+					}
+				}
+			}
+			$db_version = $db_version[0];
 			$updates = $this->__findUpgrades($db_version['AdminSetting']['value']);
 			if (!empty($updates)) {
 				foreach ($updates as $update => $temp) {

app/Model/Log.php

@@ -43,7 +43,8 @@ class Log extends AppModel {
 							'merge',
 							'undelete',
 							'file_upload',
-							'export'
+							'export',
+							'pruneUpdateLogs'
 						)),
 			'message' => 'Options : ...'
 		)
@@ -140,4 +141,69 @@ class Log extends AppModel {
 				'model_id' => $model_id,
 		));
 	}
+
+	// to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version
+	// this function remedies a fixed upgrade bug instance by eliminating the massive number of erroneous upgrade log entries
+	public function pruneUpdateLogs($jobId = false, $user) {
+		$max = $this->find('first', array('fields' => array('MAX(id) AS lastid')));
+		if (!empty($max)) {
+			$max = $max[0]['lastid'];
+		}
+		if ($jobId) {
+			$this->Job = ClassRegistry::init('Job');
+			$this->Job->id = $jobId;
+			if (!$this->Job->exists()) {
+				$jobId = false;
+			}
+		}
+		$iterations = ($max / 1000);
+		for ($i = 0; $i < $iterations; $i++) {
+			$this->deleteAll(array('action' => 'update_database', 'id >' => $i * 1000, 'id <' => ($i+1) * 1000));
+			if ($jobId) {
+				$this->Job->saveField('progress', $i * 100 / $iterations);
+			}
+		}
+		$this->create();
+		$this->save(array(
+				'org' => $user['Organisation']['name'],
+				'email' =>$user['email'],
+				'user_id' => $user['id'],
+				'action' => 'pruneUpdateLogs',
+				'title' => 'Pruning updates',
+				'change' => 'Pruning completed in ' . $i . ' iteration(s).',
+				'model' => 'Log',
+				'model_id' => 0
+		));
+	}
+
+
+	public function pruneUpdateLogsRouter($user) {
+		if (Configure::read('MISP.background_jobs')) {
+			$job = ClassRegistry::init('Job');
+			$job->create();
+			$data = array(
+					'worker' => 'default',
+					'job_type' => 'prune_update_logs',
+					'job_input' => 'All update entries',
+					'status' => 0,
+					'retries' => 0,
+					'org_id' => $user['org_id'],
+					'org' => $user['Organisation']['name'],
+					'message' => 'Purging the heretic.',
+			);
+			$job->save($data);
+			$jobId = $job->id;
+			$process_id = CakeResque::enqueue(
+					'default',
+					'AdminShell',
+					array('prune_update_logs', $jobId, $user['id']),
+					true
+			);
+			$job->saveField('process_id', $process_id);
+			return $process_id;
+		} else {
+			$result = $this->pruneUpdateLogs(false, $user);
+			return $result;
+		}
+	}
 }

app/View/Elements/healthElements/diagnostics.ctp

@@ -1,9 +1,9 @@
 <div style="border:1px solid #dddddd; margin-top:1px; width:95%; padding:10px">
-<?php 
+<?php
 	if (!$dbEncodingStatus):
 ?>
-		<div style="font-size:12pt;padding-left:3px;width:100%;background-color:red;color:white;font-weight:bold;">Incorrect database encoding setting: Your database connection is currently NOT set to UTF-8. Please make sure to uncomment the 'encoding' => 'utf8' line in <?php echo APP; ?>Config/database.php</div>	
-<?php 
+		<div style="font-size:12pt;padding-left:3px;width:100%;background-color:red;color:white;font-weight:bold;">Incorrect database encoding setting: Your database connection is currently NOT set to UTF-8. Please make sure to uncomment the 'encoding' => 'utf8' line in <?php echo APP; ?>Config/database.php</div>
+<?php
 	endif;
 ?>
 	<h3>MISP version</h3>
@@ -96,7 +96,7 @@
 			}
 		?>
 	</div>
-	
+
 	<h3>PHP Settings</h3>
 	<?php
 		$phpcolour = 'green';
@@ -145,26 +145,26 @@
 		endforeach;
 	?>
 	<h4>PHP Extensions</h4>
-		<?php 
+		<?php
 			foreach (array('web', 'cli') as $context):
 		?>
 			<div style="background-color:#f7f7f9;width:400px;">
 				<b><?php echo ucfirst(h($context));?></b><br />
-				<?php 
+				<?php
 					if (isset($extensions[$context]['extensions'])):
 						foreach ($extensions[$context]['extensions'] as $extension => $status):
 				?>
 							<?php echo h($extension); ?>:.... <span style="color:<?php echo $status ? 'green' : 'red';?>;font-weight:bold;"><?php echo $status ? 'OK' : 'Not loaded'; ?></span>
-				<?php 
+				<?php
 						endforeach;
 					else:
 				?>
 						<span class="red">Issues reading PHP settings. This could be due to the test script not being readable.</span>
-				<?php 
+				<?php
 					endif;
 				?>
 			</div><br />
-		<?php 
+		<?php
 			endforeach;
 		?>
 	<h3>
@@ -184,7 +184,7 @@
 						echo 'STIX and CyBox.... <span class="red">Could not read test script (stixtest.py).</span>';
 						$testReadError = true;
 					}
-				} 
+				}
 			}
 			if (!$testReadError) {
 				if ($stix['operational'] == 0) {
@@ -306,6 +306,11 @@
 	</div><br />
 	<span class="btn btn-inverse" style="padding-top:1px;padding-bottom:1px;" onClick="checkOrphanedAttributes();">Check for orphaned attributes</span><br /><br />
 	<?php echo $this->Form->postButton('Remove orphaned attributes', $baseurl . '/attributes/pruneOrphanedAttributes', $options = array('class' => 'btn btn-primary', 'style' => 'padding-top:1px;padding-bottom:1px;')); ?>
+	<h3>
+		Database cleanup scripts
+	</h3>
+	<p>If you run into an issue with an infinite upgrade loop (when upgrading from version ~2.4.50) that ends up filling your database with upgrade script log messages, run the following script.</p>
+	<?php echo $this->Form->postButton('Prune upgrade logs', $baseurl . '/logs/pruneUpdateLogs', $options = array('class' => 'btn btn-primary', 'style' => 'padding-top:1px;padding-bottom:1px;')); ?>
 	<h3>
 		Legacy Administrative Tools
 	</h3>