mirror of https://github.com/MISP/MISP
553 lines
21 KiB
PHP
553 lines
21 KiB
PHP
<?php
|
|
App::uses('Folder', 'Utility');
|
|
App::uses('File', 'Utility');
|
|
require_once 'AppShell.php';
|
|
|
|
/**
|
|
* @property Job $Job
|
|
* @property Server $Server
|
|
* @property Feed $Feed
|
|
* @property User $User
|
|
*/
|
|
class ServerShell extends AppShell
|
|
{
|
|
public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
|
|
public $tasks = array('ConfigLoad');
|
|
|
|
public function listServers()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
$res = ['servers'=>[]];
|
|
$servers = $this->Server->find('all', [
|
|
'fields' => ['Server.id', 'Server.name', 'Server.url'],
|
|
'recursive' => 0
|
|
]);
|
|
foreach ($servers as $server)
|
|
$res['servers'][] = $server['Server'];
|
|
|
|
echo json_encode($res) . PHP_EOL;
|
|
}
|
|
|
|
public function test()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
if (empty($this->args[0])) {
|
|
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Test'] . PHP_EOL);
|
|
}
|
|
|
|
$serverId = intval($this->args[0]);
|
|
$server = $this->Server->find('first', [
|
|
'conditions' => ['Server.id' => $serverId],
|
|
'recursive' => -1,
|
|
]);
|
|
if (!$server) {
|
|
die("Server with ID $serverId doesn't exists.");
|
|
}
|
|
|
|
$res = @$this->Server->runConnectionTest($server);
|
|
echo json_encode($res) . PHP_EOL;
|
|
}
|
|
|
|
public function pullAll()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
|
|
$userId = $this->args[0];
|
|
$user = $this->User->getAuthUser($userId);
|
|
if (empty($user)) {
|
|
die('User ID do not match an existing user.' . PHP_EOL);
|
|
}
|
|
|
|
if (!empty($this->args[1])) {
|
|
$technique = $this->args[1];
|
|
} else {
|
|
$technique = 'full';
|
|
}
|
|
|
|
$servers = $this->Server->find('all', array(
|
|
'conditions' => array('Server.pull' => 1),
|
|
'recursive' => -1,
|
|
'order' => 'Server.priority',
|
|
'fields' => array('Server.name', 'Server.id'),
|
|
));
|
|
|
|
foreach ($servers as $server) {
|
|
$jobId = CakeResque::enqueue(
|
|
'default',
|
|
'ServerShell',
|
|
array('pull', $userId, $server['Server']['id'], $technique)
|
|
);
|
|
$this->out("Enqueued pulling from {$server['Server']['name']} server as job $jobId");
|
|
}
|
|
}
|
|
|
|
public function pull()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
if (empty($this->args[0]) || empty($this->args[1])) {
|
|
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['pull'] . PHP_EOL);
|
|
}
|
|
$userId = $this->args[0];
|
|
$user = $this->User->getAuthUser($userId);
|
|
if (empty($user)) {
|
|
die('User ID do not match an existing user.' . PHP_EOL);
|
|
}
|
|
$serverId = $this->args[1];
|
|
if (!empty($this->args[2])) {
|
|
$technique = $this->args[2];
|
|
} else {
|
|
$technique = 'full';
|
|
}
|
|
if (!empty($this->args[3])) {
|
|
$jobId = $this->args[3];
|
|
} else {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'pull',
|
|
'job_input' => 'Server: ' . $serverId,
|
|
'status' => 0,
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'message' => 'Pulling.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
}
|
|
$force = false;
|
|
if (!empty($this->args[4]) && $this->args[4] === 'force') {
|
|
$force = true;
|
|
}
|
|
$this->Server->id = $serverId;
|
|
$server = $this->Server->read(null, $serverId);
|
|
if (!$server) {
|
|
die("Remote server with ID $serverId not found");
|
|
}
|
|
try {
|
|
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
|
|
if (is_array($result)) {
|
|
$message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
|
|
$this->Job->saveStatus($jobId, true, $message);
|
|
} else {
|
|
$message = __('ERROR: %s', $result);
|
|
$this->Job->saveStatus($jobId, false, $message);
|
|
}
|
|
} catch (Exception $e) {
|
|
$this->Job->saveStatus($jobId, false, __('ERROR: %s', $e->getMessage()));
|
|
throw $e;
|
|
}
|
|
echo $message . PHP_EOL;
|
|
}
|
|
|
|
public function push()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
if (empty($this->args[0]) || empty($this->args[1])) {
|
|
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['push'] . PHP_EOL);
|
|
}
|
|
$userId = $this->args[0];
|
|
$user = $this->User->getAuthUser($userId);
|
|
if (empty($user)) die('Invalid user.' . PHP_EOL);
|
|
$serverId = $this->args[1];
|
|
if (!empty($this->args[2])) {
|
|
$jobId = $this->args[2];
|
|
} else {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'push',
|
|
'job_input' => 'Server: ' . $serverId,
|
|
'status' => 0,
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'message' => 'Pushing.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
}
|
|
$technique = empty($this->args[3]) ? 'full' : $this->args[3];
|
|
$this->Job->read(null, $jobId);
|
|
$server = $this->Server->read(null, $serverId);
|
|
if (!$server) {
|
|
die("Remote server with ID $serverId not found");
|
|
}
|
|
App::uses('SyncTool', 'Tools');
|
|
$syncTool = new SyncTool();
|
|
$HttpSocket = $syncTool->setupHttpSocket($server);
|
|
$result = $this->Server->push($serverId, $technique, $jobId, $HttpSocket, $user);
|
|
|
|
if ($result !== true && !is_array($result)) {
|
|
$message = 'Job failed. Reason: ' . $result;
|
|
$this->Job->saveStatus($jobId, false, $message);
|
|
} else {
|
|
$message = 'Job done.';
|
|
$this->Job->saveStatus($jobId, true, $message);
|
|
}
|
|
|
|
if (isset($this->args[4])) {
|
|
$this->Task->id = $this->args[5];
|
|
$message = 'Job(s) started at ' . date('d/m/Y - H:i:s') . '.';
|
|
$this->Task->saveField('message', $message);
|
|
echo $message . PHP_EOL;
|
|
}
|
|
}
|
|
|
|
public function fetchFeed()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
if (empty($this->args[0]) || empty($this->args[1])) {
|
|
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
|
|
}
|
|
$userId = $this->args[0];
|
|
$user = $this->User->getAuthUser($userId);
|
|
if (empty($user)) {
|
|
echo 'Invalid user.';
|
|
die();
|
|
}
|
|
$feedId = $this->args[1];
|
|
if (!empty($this->args[2])) {
|
|
$jobId = $this->args[2];
|
|
} else {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'fetch_feeds',
|
|
'job_input' => 'Feed: ' . $feedId,
|
|
'status' => 0,
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'message' => 'Starting fetch from Feed.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
}
|
|
if ($feedId == 'all') {
|
|
$feedIds = $this->Feed->find('list', array(
|
|
'fields' => array('Feed.id', 'Feed.id'),
|
|
'conditions' => array('Feed.enabled' => 1)
|
|
));
|
|
$feedIds = array_values($feedIds);
|
|
$successes = 0;
|
|
$fails = 0;
|
|
foreach ($feedIds as $k => $feedId) {
|
|
$this->Job->saveProgress($jobId, 'Fetching feed: ' . $feedId, 100 * $k / count($feedIds));
|
|
$result = $this->Feed->downloadFromFeedInitiator($feedId, $user);
|
|
if ($result) {
|
|
$successes++;
|
|
} else {
|
|
$fails++;
|
|
}
|
|
}
|
|
$message = 'Job done. ' . $successes . ' feeds pulled successfully, ' . $fails . ' feeds could not be pulled.';
|
|
$this->Job->saveStatus($jobId, true, $message);
|
|
echo $message . PHP_EOL;
|
|
} else {
|
|
$temp = $this->Feed->find('first', array(
|
|
'fields' => array('Feed.id', 'Feed.id'),
|
|
'conditions' => array('Feed.enabled' => 1, 'Feed.id' => $feedId)
|
|
));
|
|
if (!empty($temp)) {
|
|
$result = $this->Feed->downloadFromFeedInitiator($feedId, $user, $jobId);
|
|
if (!$result) {
|
|
$this->Job->saveStatus($jobId, false);
|
|
echo 'Job failed.' . PHP_EOL;
|
|
} else {
|
|
$this->Job->saveStatus($jobId, true);
|
|
echo 'Job done.' . PHP_EOL;
|
|
}
|
|
} else {
|
|
$message = "Feed with ID $feedId not found.";
|
|
$this->Job->saveStatus($jobId, false, $message);
|
|
echo $message . PHP_EOL;
|
|
}
|
|
}
|
|
}
|
|
|
|
public function cacheServer()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
if (empty($this->args[0]) || empty($this->args[1])) {
|
|
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['cacheServer'] . PHP_EOL);
|
|
}
|
|
$userId = $this->args[0];
|
|
$user = $this->User->getAuthUser($userId);
|
|
if (empty($user)) die('Invalid user.' . PHP_EOL);
|
|
$scope = $this->args[1];
|
|
if (!empty($this->args[2])) {
|
|
$jobId = $this->args[2];
|
|
} else {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'cache_servers',
|
|
'job_input' => 'Server: ' . $scope,
|
|
'status' => 0,
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'message' => 'Starting server caching.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
}
|
|
$result = $this->Server->cacheServerInitiator($user, $scope, $jobId);
|
|
if ($result !== true) {
|
|
$message = 'Job Failed. Reason: ' . $result;
|
|
$this->Job->saveStatus($jobId, false, $message);
|
|
} else {
|
|
$message = 'Job done.';
|
|
$this->Job->saveStatus($jobId, true, $message);
|
|
}
|
|
echo $message . PHP_EOL;
|
|
}
|
|
|
|
public function cacheFeed()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
if (empty($this->args[0]) || empty($this->args[1])) {
|
|
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
|
|
}
|
|
$userId = $this->args[0];
|
|
$user = $this->User->getAuthUser($userId);
|
|
if (empty($user)) die('Invalid user.' . PHP_EOL);
|
|
$scope = $this->args[1];
|
|
if (!empty($this->args[2])) {
|
|
$jobId = $this->args[2];
|
|
} else {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'cache_feeds',
|
|
'job_input' => 'Feed: ' . $scope,
|
|
'status' => 0,
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'message' => 'Starting feed caching.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
}
|
|
$this->Job->read(null, $jobId);
|
|
try {
|
|
$result = $this->Feed->cacheFeedInitiator($user, $jobId, $scope);
|
|
} catch (Exception $e) {
|
|
CakeLog::error($e->getMessage());
|
|
$result = false;
|
|
}
|
|
|
|
if ($result === false) {
|
|
$message = __('Job failed. See error logs for more details.');
|
|
$this->Job->saveStatus($jobId, false, $message);
|
|
|
|
} else {
|
|
$total = $result['successes'] + $result['fails'];
|
|
$message = __n(
|
|
'%s feed from %s cached. Failed: %s',
|
|
'%s feeds from %s cached. Failed: %s',
|
|
$result['successes'], $result['successes'], $total, $result['fails']
|
|
);
|
|
if ($result['fails'] > 0) {
|
|
$message .= ' ' . __('See error logs for more details.');
|
|
}
|
|
$this->Job->saveStatus($jobId, true, $message);
|
|
}
|
|
echo $message . PHP_EOL;
|
|
}
|
|
|
|
public function enqueuePull()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
$timestamp = $this->args[0];
|
|
$userId = $this->args[1];
|
|
$taskId = $this->args[2];
|
|
$task = $this->Task->read(null, $taskId);
|
|
if ($timestamp != $task['Task']['next_execution_time']) {
|
|
return;
|
|
}
|
|
if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueuePull', $userId, $taskId);
|
|
$user = $this->User->getAuthUser($userId);
|
|
$servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('pull' => 1)));
|
|
$count = count($servers);
|
|
$failCount = 0;
|
|
foreach ($servers as $k => $server) {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'pull',
|
|
'job_input' => 'Server: ' . $server['Server']['id'],
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'org_id' => $user['org_id'],
|
|
'process_id' => 'Part of scheduled pull',
|
|
'message' => 'Pulling.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
$result = $this->Server->pull($user, $server['Server']['id'], 'full', $server, $jobId);
|
|
$this->Job->save(array(
|
|
'id' => $jobId,
|
|
'message' => 'Job done.',
|
|
'progress' => 100,
|
|
'status' => 4
|
|
));
|
|
if (is_numeric($result[0])) {
|
|
switch ($result[0]) {
|
|
case '1' :
|
|
$this->Job->saveField('message', 'Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server.');
|
|
break;
|
|
case '2' :
|
|
$this->Job->saveField('message', $result[1]);
|
|
break;
|
|
case '3' :
|
|
$this->Job->saveField('message', 'Sorry, incremental pushes are not yet implemented.');
|
|
break;
|
|
case '4' :
|
|
$this->Job->saveField('message', 'Invalid technique chosen.');
|
|
break;
|
|
|
|
}
|
|
$failCount++;
|
|
}
|
|
}
|
|
$this->Task->id = $task['Task']['id'];
|
|
$this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . $count);
|
|
}
|
|
|
|
public function enqueueFeedFetch()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
$timestamp = $this->args[0];
|
|
$userId = $this->args[1];
|
|
$taskId = $this->args[2];
|
|
$task = $this->Task->read(null, $taskId);
|
|
if ($timestamp != $task['Task']['next_execution_time']) {
|
|
return;
|
|
}
|
|
if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueFeedFetch', $userId, $taskId);
|
|
$user = $this->User->getAuthUser($userId);
|
|
$failCount = 0;
|
|
$feeds = $this->Feed->find('all', array(
|
|
'recursive' => -1,
|
|
'conditions' => array('enabled' => true)
|
|
));
|
|
foreach ($feeds as $k => $feed) {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'feed_fetch',
|
|
'job_input' => 'Feed: ' . $feed['Feed']['id'],
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'org_id' => $user['org_id'],
|
|
'process_id' => 'Part of scheduled feed fetch',
|
|
'message' => 'Pulling.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
$result = $this->Feed->downloadFromFeedInitiator($feed['Feed']['id'], $user, $jobId);
|
|
$this->Job->save(array(
|
|
'message' => 'Job done.',
|
|
'progress' => 100,
|
|
'status' => 4
|
|
));
|
|
if ($result !== true) {
|
|
$this->Job->saveField('message', 'Could not fetch feed.');
|
|
$failCount++;
|
|
}
|
|
}
|
|
$this->Task->id = $task['Task']['id'];
|
|
$this->Task->saveField('message', count($feeds) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '. Failed jobs: ' . $failCount . '/' . count($feeds));
|
|
}
|
|
|
|
public function enqueueFeedCache()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
$timestamp = $this->args[0];
|
|
$userId = $this->args[1];
|
|
$taskId = $this->args[2];
|
|
$task = $this->Task->read(null, $taskId);
|
|
if ($timestamp != $task['Task']['next_execution_time']) {
|
|
return;
|
|
}
|
|
if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueueFeedCache', $userId, $taskId);
|
|
$user = $this->User->getAuthUser($userId);
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'feed_cache',
|
|
'job_input' => '',
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'org_id' => $user['org_id'],
|
|
'process_id' => 'Part of scheduled feed caching',
|
|
'message' => 'Caching.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
try {
|
|
$result = $this->Feed->cacheFeedInitiator($user, $jobId, 'all');
|
|
} catch (Exception $e) {
|
|
CakeLog::error($e->getMessage());
|
|
$result = false;
|
|
}
|
|
|
|
if ($result === false) {
|
|
$message = __('Job failed. See error logs for more details.');
|
|
$this->Job->saveStatus($jobId, false, $message);
|
|
|
|
} else {
|
|
$total = $result['successes'] + $result['fails'];
|
|
$message = __n(
|
|
'%s feed from %s cached. Failed: %s',
|
|
'%s feeds from %s cached. Failed: %s',
|
|
$result['successes'], $total, $result['fails']
|
|
);
|
|
if ($result['fails'] > 0) {
|
|
$message .= ' ' . __('See error logs for more details.');
|
|
}
|
|
$this->Job->saveStatus($jobId, true, $message);
|
|
}
|
|
|
|
$this->Task->id = $task['Task']['id'];
|
|
$this->Task->saveField('message', 'Job completed at ' . date('d/m/Y - H:i:s'));
|
|
}
|
|
|
|
public function enqueuePush()
|
|
{
|
|
$this->ConfigLoad->execute();
|
|
$timestamp = $this->args[0];
|
|
$taskId = $this->args[1];
|
|
$userId = $this->args[2];
|
|
$this->Task->id = $taskId;
|
|
$task = $this->Task->read(null, $taskId);
|
|
if ($timestamp != $task['Task']['next_execution_time']) {
|
|
return;
|
|
}
|
|
if ($task['Task']['timer'] > 0) $this->Task->reQueue($task, 'default', 'ServerShell', 'enqueuePush', $userId, $taskId);
|
|
|
|
$this->User->recursive = -1;
|
|
$user = $this->User->getAuthUser($userId);
|
|
$servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('push' => 1)));
|
|
foreach ($servers as $k => $server) {
|
|
$this->Job->create();
|
|
$data = array(
|
|
'worker' => 'default',
|
|
'job_type' => 'push',
|
|
'job_input' => 'Server: ' . $server['Server']['id'],
|
|
'retries' => 0,
|
|
'org' => $user['Organisation']['name'],
|
|
'org_id' => $user['org_id'],
|
|
'process_id' => 'Part of scheduled push',
|
|
'message' => 'Pushing.',
|
|
);
|
|
$this->Job->save($data);
|
|
$jobId = $this->Job->id;
|
|
App::uses('SyncTool', 'Tools');
|
|
$syncTool = new SyncTool();
|
|
$HttpSocket = $syncTool->setupHttpSocket($server);
|
|
$this->Server->push($server['Server']['id'], 'full', $jobId, $HttpSocket, $user);
|
|
}
|
|
$this->Task->id = $task['Task']['id'];
|
|
$this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.');
|
|
}
|
|
}
|