MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Better error messages when fetching feeds

pull/7900/head
Jakub Onderka 2021-10-29 11:59:08 +02:00
parent 339a594daf
commit b9ab3332a2
2 changed files with 39 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
app/Console/Command/ServerShell.php
View File

@ -213,25 +213,13 @@ class ServerShell extends AppShell
if (!empty($this->args[2])) {
$jobId = $this->args[2];
} else {
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'fetch_feeds',
'job_input' => 'Feed: ' . $feedId,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
'message' => 'Starting fetch from Feed.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
$jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'fetch_feeds', 'Feed: ' . $feedId, 'Starting fetch from Feed.');
}
if ($feedId == 'all') {
$feedIds = $this->Feed->find('list', array(
'fields' => array('Feed.id', 'Feed.id'),
if ($feedId === 'all') {
$feedIds = $this->Feed->find('column', array(
'fields' => array('Feed.id'),
'conditions' => array('Feed.enabled' => 1)
));
$feedIds = array_values($feedIds);
$successes = 0;
$fails = 0;
foreach ($feedIds as $k => $feedId) {
@ -247,21 +235,21 @@ class ServerShell extends AppShell
$this->Job->saveStatus($jobId, true, $message);
echo $message . PHP_EOL;
} else {
$temp = $this->Feed->find('first', array(
'fields' => array('Feed.id', 'Feed.id'),
'conditions' => array('Feed.enabled' => 1, 'Feed.id' => $feedId)
));
if (!empty($temp)) {
$feedEnabled = $this->Feed->hasAny([
'Feed.enabled' => 1,
'Feed.id' => $feedId,
]);
if ($feedEnabled) {
$result = $this->Feed->downloadFromFeedInitiator($feedId, $user, $jobId);
if (!$result) {
$this->Job->saveStatus($jobId, false);
$this->Job->saveStatus($jobId, false, 'Job failed. See error log for more details.');
echo 'Job failed.' . PHP_EOL;
} else {
$this->Job->saveStatus($jobId, true);
echo 'Job done.' . PHP_EOL;
}
} else {
$message = "Feed with ID $feedId not found.";
$message = "Feed with ID $feedId not found or not enabled.";
$this->Job->saveStatus($jobId, false, $message);
echo $message . PHP_EOL;
}

42
app/Model/Feed.php
View File

@ -387,8 +387,8 @@ class Feed extends AppModel
}
$resultArray = $complexTypeTool->checkComplexRouter($data, $type, $settings);
$this->Attribute = ClassRegistry::init('Attribute');
foreach ($resultArray as $key => $value) {
$resultArray[$key]['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category'];
foreach ($resultArray as &$value) {
$value['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category'];
}
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
@ -1056,19 +1056,32 @@ class Feed extends AppModel
return $success;
}
/**
* @param int $feedId
* @param array $user
* @param int|false $jobId
* @return array|bool
* @throws Exception
*/
public function downloadFromFeedInitiator($feedId, $user, $jobId = false)
{
$this->id = $feedId;
$this->read();
if (isset($this->data['Feed']['settings']) && !empty($this->data['Feed']['settings'])) {
$this->data['Feed']['settings'] = json_decode($this->data['Feed']['settings'], true);
$feed = $this->find('first', array(
'conditions' => ['Feed.id' => $feedId],
'recursive' => -1,
));
if (empty($feed)) {
throw new Exception("Feed with ID $feedId not found.");
}
$HttpSocket = $this->isFeedLocal($this->data) ? null : $this->__setupHttpSocket();
if ($this->data['Feed']['source_format'] === 'misp') {
if (!empty($feed['Feed']['settings'])) {
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);
}
$HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket();
if ($feed['Feed']['source_format'] === 'misp') {
$this->jobProgress($jobId, 'Fetching event manifest.');
try {
$actions = $this->getNewEventUuids($this->data, $HttpSocket);
$actions = $this->getNewEventUuids($feed, $HttpSocket);
} catch (Exception $e) {
$this->logException("Could not get new event uuids for feed $feedId.", $e);
$this->jobProgress($jobId, 'Could not fetch event manifest. See error log for more details.');
@ -1081,12 +1094,12 @@ class Feed extends AppModel
$total = count($actions['add']) + count($actions['edit']);
$this->jobProgress($jobId, __("Fetching %s events.", $total));
$result = $this->downloadFromFeed($actions, $this->data, $HttpSocket, $user, $jobId);
$this->__cleanupFile($this->data, '/manifest.json');
$result = $this->downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId);
$this->__cleanupFile($feed, '/manifest.json');
} else {
$this->jobProgress($jobId, 'Fetching data.');
try {
$temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all');
$temp = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all');
} catch (Exception $e) {
$this->logException("Could not get freetext feed $feedId", $e);
$this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.');
@ -1106,17 +1119,18 @@ class Feed extends AppModel
'to_ids' => $value['to_ids']
);
}
unset($temp);
$this->jobProgress($jobId, 'Saving data.', 50);
try {
$result = $this->saveFreetextFeedData($this->data, $data, $user, $jobId);
$result = $this->saveFreetextFeedData($feed, $data, $user, $jobId);
} catch (Exception $e) {
$this->logException("Could not save freetext feed data for feed $feedId.", $e);
return false;
}
$this->__cleanupFile($this->data, '');
$this->__cleanupFile($feed, '');
}
return $result;
}