fix: Various fixes to the cached exports

- Tightened the rules for export generation when no valid published events exist
- Corrected various issues with the progress bars
- Added the missing JSON export to the caches
- XML/JSON caches now correctly take into account the cached attachent inclusion setting
- MISP will now show the users browsing the export page whether attachments will be cached with the current settings or not
- Added correct progress bar to the HIDS export

app/Console/Command/EventShell.php

@@ -54,7 +54,7 @@ class EventShell extends AppShell
 		$file->write('<?xml version="1.0" encoding="UTF-8"?>' . PHP_EOL . '<response>');
 		if (!empty($eventIds)) {
 			foreach ($eventIds as $k => $eventId) {
-				$temp = $this->Event->fetchEvent($user, array('eventid' => $eventId['Event']['id']));
+				$temp = $this->Event->fetchEvent($user, array('eventid' => $eventId['Event']['id'], 'includeAttachments' => Configure::read('MISP.cached_attachments')));
 				$file->append($converter->event2XML($temp[0], $user['Role']['perm_site_admin']) . PHP_EOL);
 				$this->Job->saveField('progress', ($k+1) / $eventCount *100);
 			}
@@ -67,6 +67,37 @@ class EventShell extends AppShell
 		$this->Job->saveField('message', 'Job done. (in '.$timeDelta.'s)');
 		$this->Job->saveField('date_modified', date("y-m-d H:i:s"));
 	}
+	
+	public function cachejson() {
+		$timeStart = time();
+		$userId = $this->args[0];
+		$id = $this->args[1];
+		$user = $this->User->getAuthUser($userId);
+		$this->Job->id = $id;
+		// TEMP: change to passing an options array with the user!!
+		$eventIds = $this->Event->fetchEventIds($user);
+		$eventCount = count($eventIds);
+		$dir = new Folder(APP . 'tmp/cached_exports/json', true, 0750);
+		if ($user['Role']['perm_site_admin']) {
+			$file = new File($dir->pwd() . DS . 'misp.json' . '.ADMIN.json');
+		} else {
+			$file = new File($dir->pwd() . DS . 'misp.json' . '.' . $user['Organisation']['name'] . '.json');
+		}
+		App::uses('JSONConverterTool', 'Tools');
+		$converter = new JSONConverterTool();
+		$file->append('{"response":[');
+		foreach ($eventIds as $k => $eventId) {
+			$result = $this->Event->fetchEvent($user, array('eventid' => $eventId['Event']['id'], 'includeAttachments' => Configure::read('MISP.cached_attachments')));
+			$file->append($converter->event2JSON($result[0]));
+			if ($k < count($eventIds) -1 ) $file->append(',');
+		}
+		$file->append(']}');
+		$file->close();
+		$timeDelta = (time()-$timeStart);
+		$this->Job->saveField('progress', 100);
+		$this->Job->saveField('message', 'Job done. (in '.$timeDelta.'s)');
+		$this->Job->saveField('date_modified', date("y-m-d H:i:s"));
+	}
 
 	private function __recursiveEcho($array) {
 		$text = "";
@@ -99,7 +130,7 @@ class EventShell extends AppShell
 		$this->Job->id = $id;
 		$extra = $this->args[2];
 		$this->Job->saveField('progress', 1);
-		$rules = $this->Attribute->hids($user, $extra);
+		$rules = $this->Attribute->hids($user, $extra, '', false, false, false, $id);
 		$this->Job->saveField('progress', 80);
 		$dir = new Folder(APP . DS . '/tmp/cached_exports/' . $extra, true, 0750);
 		if ($user['Role']['perm_site_admin']) {
@@ -108,8 +139,12 @@ class EventShell extends AppShell
 			$file = new File($dir->pwd() . DS . 'misp.' . $extra . '.' . $user['Organisation']['name'] . '.txt');
 		}
 		$file->write('');
-		foreach ($rules as $rule) {
-			$file->append($rule . PHP_EOL);
+		if (!empty($rules)) {
+			foreach ($rules as $rule) {
+				$file->append($rule . '\n');
+			}
+		} else {
+			$file->append("No exportable " . $type . "s found. " . '\n');
 		}
 		$file->close();
 		$timeDelta = (time()-$timeStart);

app/Controller/EventsController.php

@@ -1552,7 +1552,7 @@ class EventsController extends AppController {
 			} else {
 				$useOrg = $this->Auth->user('Organisation')['name'];
 				$useOrg_id = $this->Auth->user('org_id');
-				$conditions['OR'][] = array('id' => $this->Event->fetchEventIds($this->Auth->user, false, false, true, true));
+				$conditions['OR'][] = array('id' => $this->Event->fetchEventIds($this->Auth->user(), false, false, false, true));
 			}
 			$this->Event->recursive = -1;
 			$newestEvent = $this->Event->find('first', array(
@@ -1560,8 +1560,18 @@ class EventsController extends AppController {
 				'fields' => 'timestamp',
 				'order' => 'Event.timestamp DESC',
 			));
+			$newestEventPublished = $this->Event->find('first', array(
+				'conditions' => array('AND' => array($conditions, array('published' => 1))),
+				'fields' => 'timestamp',
+				'order' => 'Event.timestamp DESC',
+			));
 			$this->loadModel('Job');
 			foreach ($this->Event->export_types as $k => $type) {
+				if ($type['requiresPublished']) {
+					$tempNewestEvent = $newestEventPublished;
+				} else {
+					$tempNewestEvent = $newestEvent;
+				}
 				$job = $this->Job->find('first', array(
 						'fields' => array('id', 'progress'),
 						'conditions' => array(
@@ -1578,12 +1588,20 @@ class EventsController extends AppController {
 					$file = new File($dir->pwd() . DS . 'misp.' . $k . '.' . $useOrg . $type['extension']);
 				}
 				if (!$file->readable()) {
-					$lastModified = 'N/A';
-					$this->Event->export_types[$k]['recommendation'] = 1;
+					if (empty($tempNewestEvent)) {
+						$lastModified = 'No valid events';
+						$this->Event->export_types[$k]['recommendation'] = 0;
+					} else {
+						$lastModified = 'N/A';
+						$this->Event->export_types[$k]['recommendation'] = 1;
+					}
 				} else {
 					$fileChange = $file->lastChange();
 					$lastModified = $this->__timeDifference($now, $fileChange);
-					if ($fileChange > $newestEvent['Event']['timestamp']) {
+					if (empty($tempNewestEvent) || $fileChange > $tempNewestEvent['Event']['timestamp']) {
+						if (empty($tempNewestEvent)) {
+							$lastModified = 'No valid events';
+						}
 						$this->Event->export_types[$k]['recommendation'] = 0;
 					} else {
 						$this->Event->export_types[$k]['recommendation'] = 1;

app/Model/Attribute.php

@@ -1268,7 +1268,7 @@ class Attribute extends AppModel {
 	}
 
 
-	public function hids($user, $type, $tags = '', $from = false, $to = false, $last = false) {
+	public function hids($user, $type, $tags = '', $from = false, $to = false, $last = false, $jobId = false) {
 		if (empty($user)) throw new MethodNotAllowedException('Could not read user.');
 		// check if it's a valid type
 		if ($type != 'md5' && $type != 'sha1' && $type != 'sha256') {
@@ -1296,7 +1296,13 @@ class Attribute extends AppModel {
 		}
 		App::uses('HidsExport', 'Export');
 		$continue = false;
-		foreach ($eventIds as $event) {
+		$eventCount = count($eventIds);
+		if ($jobId) {
+			$this->Job = ClassRegistry::init('Job');
+			$this->Job->id = $jobId;
+			if (!$this->Job->exists()) $jobId = false;
+		}
+		foreach ($eventIds as $k => $event) {
 			$conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1, 'Attribute.type' => $typeArray, 'Attribute.event_id' => $event['Event']['id']);
 			$options = array(
 					'conditions' => $conditions,
@@ -1307,6 +1313,9 @@ class Attribute extends AppModel {
 			$export = new HidsExport();
 			$rules = array_merge($rules, $export->export($items, strtoupper($type), $continue));
 			$continue = true;
+			if ($jobId && ($k % 10 == 0)) {
+				$this->Job->saveField('progress', $k * 80 / $eventCount);
+			}
 		}
 		return $rules;
 	}

app/Model/Event.php

@@ -71,49 +71,74 @@ class Event extends AppModel {
 	public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group');
 
 	public $export_types = array(
+			'json' => array(
+					'extension' => '.json',
+					'type' => 'JSON',
+					'requiresPublished' => 0,
+					'canHaveAttachments' => true,
+					'description' => 'Click this to download all events and attributes that you have access to in MISP XML format.',
+			),
 			'xml' => array(
 					'extension' => '.xml',
 					'type' => 'XML',
-					'description' => 'Click this to download all events and attributes that you have access to <small>(except file attachments)</small> in a custom XML format.',
+					'requiresPublished' => 0,
+					'canHaveAttachments' => true,
+					'description' => 'Click this to download all events and attributes that you have access to in MISP JSON format.',
 			),
 			'csv_sig' => array(
 					'extension' => '.csv',
 					'type' => 'CSV_Sig',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click this to download all attributes that are indicators and that you have access to <small>(except file attachments)</small> in CSV format.',
 			),
 			'csv_all' => array(
 					'extension' => '.csv',
 					'type' => 'CSV_All',
+					'requiresPublished' => 0,
+					'canHaveAttachments' => false,
 					'description' => 'Click this to download all attributes that you have access to <small>(except file attachments)</small> in CSV format.',
 			),
 			'suricata' => array(
 					'extension' => '.rules',
 					'type' => 'Suricata',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click this to download all network related attributes that you have access to under the Suricata rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
 			),
 			'snort' => array(
 					'extension' => '.rules',
 					'type' => 'Snort',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
 			),
 			'rpz' => array(
 					'extension' => '.txt',
 					'type' => 'RPZ',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click this to download an RPZ Zone file generated from all ip-src/ip-dst, hostname, domain attributes. This can be useful for DNS level firewalling. Only published events and attributes marked as IDS Signature are exported.'
 			),
 			'md5' => array(
 					'extension' => '.txt',
 					'type' => 'MD5',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click on one of these two buttons to download all MD5 checksums contained in file-related attributes. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.',
 			),
 			'sha1' => array(
 					'extension' => '.txt',
 					'type' => 'SHA1',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click on one of these two buttons to download all SHA1 checksums contained in file-related attributes. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.',
 			),
 			'text' => array(
 					'extension' => '.txt',
 					'type' => 'TEXT',
+					'requiresPublished' => 1,
+					'canHaveAttachments' => false,
 					'description' => 'Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.'
 			),
 	);
@@ -1048,6 +1073,7 @@ class Event extends AppModel {
 		// restricting to non-private or same org if the user is not a site-admin.
 		if (!$user['Role']['perm_site_admin']) {
 			$sgids = $this->SharingGroup->fetchAllAuthorised($user);
+			if (empty($sgids)) $sgids = -1;
 			$conditions['AND']['OR'] = array(
 				'Event.org_id' => $user['org_id'],
 				array(
@@ -1073,7 +1099,6 @@ class Event extends AppModel {
 		if ($last) $conditions['AND'][] = array('Event.publish_timestamp >=' => $last);
 		if ($timestamp) $conditions['AND'][] = array('Event.timestamp >=' => $timestamp);
 		if ($publish_timestamp) $conditions['AND'][] = array('Event.publish_timestamp >=' => $publish_timestamp);
-
 		if ($list) {
 			$params = array(
 				'conditions' => $conditions,

app/Model/Job.php

@@ -38,7 +38,7 @@ class Job extends AppModel {
 				'job_input' => $target,
 				'status' => 0,
 				'retries' => 0,
-				'org_id' => $user['org_id'],
+				'org_id' => $user['Role']['perm_site_admin'] ? 0 : $user['org_id'],
 				'message' => 'Fetching events.',
 		);
 		if ($type === 'md5' || $type === 'sha1') {

app/View/Events/export.ctp

@@ -33,7 +33,22 @@
 			<tr>
 				<td class="short"><?php echo $type['type']; ?></td>
 				<td id="update<?php echo $i; ?>" class="short" style="color:red;"><?php echo $type['lastModified']; ?></td>
-				<td><?php echo $type['description']; ?></td>
+				<td>
+					<?php 
+						echo $type['description'];
+						if ($type['canHaveAttachments']):
+							if (Configure::read('MISP.cached_attachments')):
+					?>
+						<span class="green"> (Attachments are enabled on this instance)</span>
+					<?php 
+							else:
+					?>
+						<span class="red"> (Attachments are disabled on this instance)</span>
+					<?php 
+							endif;
+						endif;
+					?>
+				</td>
 				<td id="outdated<?php echo $i; ?>">
 					<?php
 						if ($type['recommendation']) {

app/View/Events/hids.ctp

@@ -1,8 +1,5 @@
 <?php
 	if (count($rules) >= 1) {
-		echo ("#This part is not finished and might be buggy. Please report any issues." . PHP_EOL);
-
-		echo "# " . PHP_EOL;
 		foreach ($rules as &$rule)
 			echo $rule .  PHP_EOL;
 		echo "#" . PHP_EOL;