2016-06-04 01:10:45 +02:00
< ? php
2014-01-06 05:15:47 +01:00
App :: uses ( 'Folder' , 'Utility' );
App :: uses ( 'File' , 'Utility' );
require_once 'AppShell.php' ;
2020-06-26 17:14:18 +02:00
/**
* @ property Job $Job
* @ property Server $Server
* @ property Feed $Feed
* @ property User $User
*/
2014-01-06 05:15:47 +01:00
class ServerShell extends AppShell
{
2022-01-14 14:54:39 +01:00
public $uses = array ( 'Server' , 'Task' , 'Job' , 'User' , 'Feed' , 'TaxiiServer' );
2016-06-04 01:08:16 +02:00
2022-07-11 14:34:17 +02:00
public function getOptionParser ()
{
$parser = parent :: getOptionParser ();
$parser -> addSubcommand ( 'fetchIndex' , [
'help' => __ ( 'Fetch remote instance event index.' ),
'parser' => array (
'arguments' => array (
'server_id' => [ 'help' => __ ( 'Remote server ID.' ), 'required' => true ],
),
)
]);
return $parser ;
}
2021-03-03 00:29:57 +01:00
public function list ()
{
$servers = $this -> Server -> find ( 'all' , [
'fields' => [ 'Server.id' , 'Server.name' , 'Server.url' ],
'recursive' => 0
]);
foreach ( $servers as $server ) {
echo sprintf (
'%sServer #%s :: %s :: %s' ,
PHP_EOL ,
$server [ 'Server' ][ 'id' ],
$server [ 'Server' ][ 'name' ],
$server [ 'Server' ][ 'url' ]
);
}
echo PHP_EOL ;
}
2020-02-10 15:15:59 +01:00
public function listServers ()
{
2019-10-22 20:13:11 +02:00
$servers = $this -> Server -> find ( 'all' , [
'fields' => [ 'Server.id' , 'Server.name' , 'Server.url' ],
'recursive' => 0
]);
2021-09-03 08:48:44 +02:00
$res = [ 'servers' => array_column ( $servers , 'Server' )];
echo $this -> json ( $res ) . PHP_EOL ;
2019-10-22 20:13:11 +02:00
}
2020-02-10 15:15:59 +01:00
public function test ()
{
2019-10-22 20:13:11 +02:00
if ( empty ( $this -> args [ 0 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Test' ] . PHP_EOL );
}
$serverId = intval ( $this -> args [ 0 ]);
2021-09-03 08:48:44 +02:00
$server = $this -> getServer ( $serverId );
2019-10-22 20:13:11 +02:00
2021-08-21 11:20:33 +02:00
$res = $this -> Server -> runConnectionTest ( $server , false );
2021-09-03 08:48:44 +02:00
echo $this -> json ( $res ) . PHP_EOL ;
2019-10-22 20:13:11 +02:00
}
2022-07-11 14:34:17 +02:00
public function fetchIndex ()
{
$serverId = intval ( $this -> args [ 0 ]);
$server = $this -> getServer ( $serverId );
$serverSync = new ServerSyncTool ( $server , $this -> Server -> setupSyncRequest ( $server ));
$index = $this -> Server -> getEventIndexFromServer ( $serverSync );
echo $this -> json ( $index ) . PHP_EOL ;
}
2020-06-09 10:19:22 +02:00
public function pullAll ()
{
2021-03-18 23:44:04 +01:00
if ( empty ( $this -> args [ 0 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'PullAll' ] . PHP_EOL );
}
2020-06-09 10:19:22 +02:00
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2024-02-07 11:10:35 +01:00
Configure :: write ( 'CurrentUserId' , $userId );
2020-06-09 10:19:22 +02:00
if ( ! empty ( $this -> args [ 1 ])) {
$technique = $this -> args [ 1 ];
} else {
$technique = 'full' ;
}
2021-09-03 08:48:44 +02:00
$servers = $this -> Server -> find ( 'list' , array (
2020-06-09 10:19:22 +02:00
'conditions' => array ( 'Server.pull' => 1 ),
'recursive' => - 1 ,
'order' => 'Server.priority' ,
2021-09-03 08:48:44 +02:00
'fields' => array ( 'Server.id' , 'Server.name' ),
2020-06-09 10:19:22 +02:00
));
2021-09-03 08:48:44 +02:00
foreach ( $servers as $serverId => $serverName ) {
2021-11-22 10:27:15 +01:00
$jobId = $this -> Job -> createJob ( $user , Job :: WORKER_DEFAULT , 'pull' , " Server: $serverId " , 'Pulling.' );
2021-11-20 14:49:58 +01:00
$backgroundJobId = $this -> getBackgroundJobsTool () -> enqueue (
2021-10-27 17:03:16 +02:00
BackgroundJobsTool :: DEFAULT_QUEUE ,
BackgroundJobsTool :: CMD_SERVER ,
[
'pull' ,
$user [ 'id' ],
$serverId ,
2021-11-22 10:27:15 +01:00
$technique ,
$jobId ,
],
true ,
$jobId
2020-06-09 10:19:22 +02:00
);
2021-10-27 17:03:16 +02:00
2021-10-29 14:52:42 +02:00
$this -> out ( " Enqueued pulling from $serverName server as job $backgroundJobId " );
2020-06-09 10:19:22 +02:00
}
}
2020-02-10 15:15:59 +01:00
public function pull ()
{
2019-08-08 11:18:21 +02:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ])) {
2021-03-18 23:44:04 +01:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Pull' ] . PHP_EOL );
2019-08-08 11:18:21 +02:00
}
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2019-08-08 11:18:21 +02:00
$serverId = $this -> args [ 1 ];
2021-09-03 08:48:44 +02:00
$server = $this -> getServer ( $serverId );
2024-02-07 11:10:35 +01:00
Configure :: write ( 'CurrentUserId' , $userId );
2019-08-08 11:18:21 +02:00
if ( ! empty ( $this -> args [ 2 ])) {
$technique = $this -> args [ 2 ];
} else {
$technique = 'full' ;
}
if ( ! empty ( $this -> args [ 3 ])) {
$jobId = $this -> args [ 3 ];
} else {
2021-10-29 14:52:42 +02:00
$jobId = $this -> Job -> createJob ( $user , Job :: WORKER_DEFAULT , 'pull' , 'Server: ' . $serverId , 'Pulling.' );
2019-08-08 11:18:21 +02:00
}
2020-05-14 09:34:49 +02:00
$force = false ;
if ( ! empty ( $this -> args [ 4 ]) && $this -> args [ 4 ] === 'force' ) {
$force = true ;
}
2024-04-08 19:45:30 +02:00
// Try to enable garbage collector as pulling events can use a lot of memory
gc_enable ();
2020-06-26 17:14:18 +02:00
try {
2021-09-30 23:19:29 +02:00
$result = $this -> Server -> pull ( $user , $technique , $server , $jobId , $force );
2020-06-26 17:14:18 +02:00
if ( is_array ( $result )) {
2024-02-15 14:39:43 +01:00
$message = __ ( 'Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.' , count ( $result [ 0 ]), count ( $result [ 1 ]), $result [ 2 ], $result [ 3 ], $result [ 4 ], $result [ 5 ]);
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , true , $message );
} else {
$message = __ ( 'ERROR: %s' , $result );
$this -> Job -> saveStatus ( $jobId , false , $message );
}
} catch ( Exception $e ) {
$this -> Job -> saveStatus ( $jobId , false , __ ( 'ERROR: %s' , $e -> getMessage ()));
throw $e ;
2019-08-08 11:18:21 +02:00
}
echo $message . PHP_EOL ;
}
2016-06-04 01:08:16 +02:00
2020-02-10 15:15:59 +01:00
public function push ()
{
2019-08-08 11:18:21 +02:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ])) {
2021-03-18 23:44:04 +01:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Push' ] . PHP_EOL );
2019-08-08 11:18:21 +02:00
}
2024-04-17 15:08:38 +02:00
2019-08-08 11:18:21 +02:00
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2019-08-08 11:18:21 +02:00
$serverId = $this -> args [ 1 ];
2021-09-03 08:48:44 +02:00
$server = $this -> getServer ( $serverId );
2024-02-07 11:10:35 +01:00
Configure :: write ( 'CurrentUserId' , $userId );
2021-10-29 14:52:42 +02:00
$technique = empty ( $this -> args [ 2 ]) ? 'full' : $this -> args [ 2 ];
if ( ! empty ( $this -> args [ 3 ])) {
$jobId = $this -> args [ 3 ];
2019-08-08 11:18:21 +02:00
} else {
2021-10-29 14:52:42 +02:00
$jobId = $this -> Job -> createJob ( $user , Job :: WORKER_DEFAULT , 'push' , 'Server: ' . $serverId , 'Pushing.' );
2019-08-08 11:18:21 +02:00
}
$this -> Job -> read ( null , $jobId );
2021-09-03 08:48:44 +02:00
2019-08-08 11:18:21 +02:00
App :: uses ( 'SyncTool' , 'Tools' );
$syncTool = new SyncTool ();
$HttpSocket = $syncTool -> setupHttpSocket ( $server );
$result = $this -> Server -> push ( $serverId , $technique , $jobId , $HttpSocket , $user );
2020-06-26 17:14:18 +02:00
if ( $result !== true && ! is_array ( $result )) {
$message = 'Job failed. Reason: ' . $result ;
$this -> Job -> saveStatus ( $jobId , false , $message );
} else {
$message = 'Job done.' ;
$this -> Job -> saveStatus ( $jobId , true , $message );
}
2019-08-08 11:18:21 +02:00
if ( isset ( $this -> args [ 4 ])) {
$this -> Task -> id = $this -> args [ 5 ];
$message = 'Job(s) started at ' . date ( 'd/m/Y - H:i:s' ) . '.' ;
$this -> Task -> saveField ( 'message' , $message );
echo $message . PHP_EOL ;
}
}
2016-06-04 01:08:16 +02:00
2021-05-19 14:59:23 +02:00
public function pushAll ()
{
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2024-02-07 11:10:35 +01:00
Configure :: write ( 'CurrentUserId' , $userId );
2021-09-03 08:48:44 +02:00
$technique = isset ( $this -> args [ 1 ]) ? $this -> args [ 1 ] : 'full' ;
2021-05-19 14:59:23 +02:00
2021-09-03 08:48:44 +02:00
$servers = $this -> Server -> find ( 'list' , array (
2021-05-19 14:59:23 +02:00
'conditions' => array ( 'Server.push' => 1 ),
'recursive' => - 1 ,
'order' => 'Server.priority' ,
2021-09-03 08:48:44 +02:00
'fields' => array ( 'Server.id' , 'Server.name' ),
2021-05-19 14:59:23 +02:00
));
2021-09-03 08:48:44 +02:00
foreach ( $servers as $serverId => $serverName ) {
2021-11-20 14:49:58 +01:00
$jobId = $this -> getBackgroundJobsTool () -> enqueue (
2021-10-27 17:03:16 +02:00
BackgroundJobsTool :: DEFAULT_QUEUE ,
BackgroundJobsTool :: CMD_SERVER ,
[
'push' ,
$user [ 'id' ],
$serverId ,
$technique
]
2021-05-19 14:59:23 +02:00
);
2021-10-27 17:03:16 +02:00
2021-09-03 08:48:44 +02:00
$this -> out ( " Enqueued pushing from $serverName server as job $jobId " );
2021-05-19 14:59:23 +02:00
}
}
2022-11-04 14:02:43 +01:00
public function listFeeds ()
{
$fields = [
'id' => 3 ,
'source_format' => 10 ,
'provider' => 15 ,
'url' => 50 ,
'enabled' => 8 ,
'caching_enabled' => 7
];
$feeds = $this -> Feed -> find ( 'all' , [
'recursive' => - 1 ,
'fields' => array_keys ( $fields )
]);
2022-11-04 15:28:57 +01:00
$outputStyle = ( empty ( $this -> args [ 0 ]) || $this -> args [ 0 ] === 'json' ) ? 'json' : 'table' ;
if ( $outputStyle === 'table' ) {
$this -> out ( str_repeat ( '=' , 114 ));
2022-11-04 14:02:43 +01:00
$this -> out ( sprintf (
'| %s | %s | %s | %s | %s | %s |' ,
2022-11-04 15:28:57 +01:00
str_pad ( 'ID' , $fields [ 'id' ], ' ' , STR_PAD_RIGHT ),
str_pad ( 'Format' , $fields [ 'source_format' ], ' ' , STR_PAD_RIGHT ),
str_pad ( 'Provider' , $fields [ 'provider' ], ' ' , STR_PAD_RIGHT ),
str_pad ( 'Url' , $fields [ 'url' ], ' ' , STR_PAD_RIGHT ),
str_pad ( 'Fetching' , $fields [ 'enabled' ], ' ' , STR_PAD_RIGHT ),
str_pad ( 'Caching' , $fields [ 'caching_enabled' ], ' ' , STR_PAD_RIGHT )
2022-11-04 14:02:43 +01:00
), 1 , Shell :: NORMAL );
2022-11-04 15:28:57 +01:00
$this -> out ( str_repeat ( '=' , 114 ));
foreach ( $feeds as $feed ) {
$this -> out ( sprintf (
'| %s | %s | %s | %s | %s | %s |' ,
str_pad ( $feed [ 'Feed' ][ 'id' ], $fields [ 'id' ], ' ' , STR_PAD_RIGHT ),
str_pad ( $feed [ 'Feed' ][ 'source_format' ], $fields [ 'source_format' ], ' ' , STR_PAD_RIGHT ),
str_pad ( mb_substr ( $feed [ 'Feed' ][ 'provider' ], 0 , 13 ), $fields [ 'provider' ], ' ' , STR_PAD_RIGHT ),
str_pad (
mb_substr ( $feed [ 'Feed' ][ 'url' ], 0 , 48 ),
$fields [ 'url' ],
' ' ,
STR_PAD_RIGHT
),
$feed [ 'Feed' ][ 'enabled' ] ?
'<info>' . str_pad ( __ ( 'Yes' ), $fields [ 'enabled' ], ' ' , STR_PAD_RIGHT ) . '</info>' :
str_pad ( __ ( 'No' ), $fields [ 'enabled' ], ' ' , STR_PAD_RIGHT ),
$feed [ 'Feed' ][ 'caching_enabled' ] ?
'<info>' . str_pad ( __ ( 'Yes' ), $fields [ 'caching_enabled' ], ' ' , STR_PAD_RIGHT ) . '</info>' :
str_pad ( __ ( 'No' ), $fields [ 'caching_enabled' ], ' ' , STR_PAD_RIGHT )
), 1 , Shell :: NORMAL );
}
$this -> out ( str_repeat ( '=' , 114 ));
} else {
$this -> out ( json_encode ( $feeds , JSON_PRETTY_PRINT ));
2022-11-04 14:02:43 +01:00
}
}
public function viewFeed ()
{
2022-11-04 15:15:35 +01:00
if ( empty ( $this -> args [ 0 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'View Feed' ] . PHP_EOL );
2022-11-04 14:02:43 +01:00
}
$feed = $this -> Feed -> find ( 'first' , [
'conditions' => [
'id' => $this -> args [ 0 ]
],
'recursive' => - 1
]);
if ( empty ( $feed )) {
throw new NotFoundException ( __ ( 'Invalid feed.' ));
}
2022-11-04 15:28:57 +01:00
$outputStyle = ( empty ( $this -> args [ 1 ]) || $this -> args [ 1 ] === 'json' ) ? 'json' : 'table' ;
if ( $outputStyle === 'table' ) {
$this -> out ( str_repeat ( '=' , 114 ));
foreach ( $feed [ 'Feed' ] as $field => $value ) {
$this -> out ( sprintf (
'| %s | %s |' ,
str_pad ( $field , 20 , ' ' , STR_PAD_RIGHT ),
str_pad ( $value , 87 )
), 1 , Shell :: NORMAL );
}
$this -> out ( str_repeat ( '=' , 114 ));
} else {
$this -> out ( json_encode ( $feed ));
}
2022-11-04 14:02:43 +01:00
}
public function toggleFeed ()
{
if ( empty ( $this -> args [ 0 ])) {
2022-11-04 15:15:35 +01:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Toggle feed fetching' ] . PHP_EOL );
2022-11-04 14:02:43 +01:00
}
$feed = $this -> Feed -> find ( 'first' , [
'conditions' => [
'id' => $this -> args [ 0 ]
],
'recursive' => - 1
]);
if ( empty ( $feed )) {
throw new NotFoundException ( __ ( 'Invalid feed.' ));
}
$feed [ 'Feed' ][ 'enabled' ] = ( $feed [ 'Feed' ][ 'enabled' ]) ? 0 : 1 ;
if ( $this -> Feed -> save ( $feed )) {
$this -> out ( __ ( 'Feed fetching %s for feed %s' , ( $feed [ 'Feed' ][ 'enabled' ] ? __ ( 'enabled' ) : __ ( 'disabled' )), $feed [ 'Feed' ][ 'id' ]));
} else {
$this -> out ( __ ( 'Could not toggle fetching for feed %s' , $feed [ 'Feed' ][ 'id' ]));
}
}
public function toggleFeedCaching ()
{
if ( empty ( $this -> args [ 0 ])) {
2022-11-04 15:15:35 +01:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Toggle feed caching' ] . PHP_EOL );
2022-11-04 14:02:43 +01:00
}
$feed = $this -> Feed -> find ( 'first' , [
'conditions' => [
'id' => $this -> args [ 0 ]
],
'recursive' => - 1
]);
if ( empty ( $feed )) {
throw new NotFoundException ( __ ( 'Invalid feed.' ));
}
$feed [ 'Feed' ][ 'caching_enabled' ] = ( $feed [ 'Feed' ][ 'caching_enabled' ]) ? 0 : 1 ;
if ( $this -> Feed -> save ( $feed )) {
$this -> out ( __ ( 'Feed caching %s for feed %s' , ( $feed [ 'Feed' ][ 'enabled' ] ? __ ( 'enabled' ) : __ ( 'disabled' )), $feed [ 'Feed' ][ 'id' ]));
} else {
$this -> out ( __ ( 'Could not toggle caching for feed %s' , $feed [ 'Feed' ][ 'id' ]));
}
}
public function loadDefaultFeeds ()
{
$this -> Feed -> load_default_feeds ();
$this -> out ( __ ( 'Default feed metadata loaded.' ));
}
2020-02-10 15:15:59 +01:00
public function fetchFeed ()
{
2019-08-08 11:18:21 +02:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ])) {
2019-10-08 09:30:24 +02:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Fetch feeds as local data' ] . PHP_EOL );
2019-08-08 11:18:21 +02:00
}
2024-04-17 15:08:38 +02:00
2019-08-08 11:18:21 +02:00
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2019-08-08 11:18:21 +02:00
$feedId = $this -> args [ 1 ];
2022-09-13 16:26:29 +02:00
Configure :: write ( 'CurrentUserId' , $userId );
2019-08-08 11:18:21 +02:00
if ( ! empty ( $this -> args [ 2 ])) {
$jobId = $this -> args [ 2 ];
} else {
2021-10-29 11:59:08 +02:00
$jobId = $this -> Job -> createJob ( $user , Job :: WORKER_DEFAULT , 'fetch_feeds' , 'Feed: ' . $feedId , 'Starting fetch from Feed.' );
2019-08-08 11:18:21 +02:00
}
2021-10-29 11:59:08 +02:00
if ( $feedId === 'all' ) {
$feedIds = $this -> Feed -> find ( 'column' , array (
'fields' => array ( 'Feed.id' ),
2019-08-08 11:18:21 +02:00
'conditions' => array ( 'Feed.enabled' => 1 )
));
$successes = 0 ;
$fails = 0 ;
foreach ( $feedIds as $k => $feedId ) {
2020-06-26 17:14:18 +02:00
$this -> Job -> saveProgress ( $jobId , 'Fetching feed: ' . $feedId , 100 * $k / count ( $feedIds ));
2019-08-08 11:18:21 +02:00
$result = $this -> Feed -> downloadFromFeedInitiator ( $feedId , $user );
if ( $result ) {
$successes ++ ;
} else {
$fails ++ ;
}
}
2020-06-26 17:14:18 +02:00
$message = 'Job done. ' . $successes . ' feeds pulled successfully, ' . $fails . ' feeds could not be pulled.' ;
$this -> Job -> saveStatus ( $jobId , true , $message );
echo $message . PHP_EOL ;
2019-08-08 11:18:21 +02:00
} else {
2021-10-29 11:59:08 +02:00
$feedEnabled = $this -> Feed -> hasAny ([
'Feed.enabled' => 1 ,
'Feed.id' => $feedId ,
]);
if ( $feedEnabled ) {
2019-08-08 11:18:21 +02:00
$result = $this -> Feed -> downloadFromFeedInitiator ( $feedId , $user , $jobId );
if ( ! $result ) {
2021-10-29 11:59:08 +02:00
$this -> Job -> saveStatus ( $jobId , false , 'Job failed. See error log for more details.' );
2020-06-26 17:14:18 +02:00
echo 'Job failed.' . PHP_EOL ;
} else {
2020-07-21 11:50:31 +02:00
$this -> Job -> saveStatus ( $jobId , true );
2020-06-26 17:14:18 +02:00
echo 'Job done.' . PHP_EOL ;
2019-08-08 11:18:21 +02:00
}
2020-06-26 17:14:18 +02:00
} else {
2021-10-29 11:59:08 +02:00
$message = " Feed with ID $feedId not found or not enabled. " ;
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , false , $message );
echo $message . PHP_EOL ;
2019-08-08 11:18:21 +02:00
}
}
}
2017-05-08 14:22:27 +02:00
2020-02-10 15:15:59 +01:00
public function cacheServer ()
{
2019-01-18 09:06:23 +01:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ])) {
2021-03-18 23:44:04 +01:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Cache server' ] . PHP_EOL );
2019-01-18 09:06:23 +01:00
}
2024-04-17 15:08:38 +02:00
2019-01-18 09:06:23 +01:00
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2019-01-18 09:06:23 +01:00
$scope = $this -> args [ 1 ];
if ( ! empty ( $this -> args [ 2 ])) {
$jobId = $this -> args [ 2 ];
} else {
$this -> Job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'cache_servers' ,
2020-06-26 17:14:18 +02:00
'job_input' => 'Server: ' . $scope ,
2019-01-18 09:06:23 +01:00
'status' => 0 ,
'retries' => 0 ,
'org' => $user [ 'Organisation' ][ 'name' ],
'message' => 'Starting server caching.' ,
);
$this -> Job -> save ( $data );
$jobId = $this -> Job -> id ;
}
2019-01-18 16:15:22 +01:00
$result = $this -> Server -> cacheServerInitiator ( $user , $scope , $jobId );
2019-01-18 09:06:23 +01:00
if ( $result !== true ) {
2020-06-26 17:14:18 +02:00
$message = 'Job Failed. Reason: ' . $result ;
$this -> Job -> saveStatus ( $jobId , false , $message );
2019-01-18 09:06:23 +01:00
} else {
$message = 'Job done.' ;
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , true , $message );
2019-01-18 09:06:23 +01:00
}
echo $message . PHP_EOL ;
}
2021-05-19 14:55:19 +02:00
public function cacheServerAll ()
{
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2021-05-19 14:55:19 +02:00
2021-09-03 08:48:44 +02:00
$servers = $this -> Server -> find ( 'list' , array (
2021-05-19 14:55:19 +02:00
'conditions' => array ( 'Server.pull' => 1 ),
'recursive' => - 1 ,
'order' => 'Server.priority' ,
2021-09-03 08:48:44 +02:00
'fields' => array ( 'Server.id' , 'Server.name' ),
2021-05-19 14:55:19 +02:00
));
2021-09-03 08:48:44 +02:00
foreach ( $servers as $serverId => $serverName ) {
2021-11-20 14:49:58 +01:00
$jobId = $this -> getBackgroundJobsTool () -> enqueue (
2021-10-27 17:03:16 +02:00
BackgroundJobsTool :: DEFAULT_QUEUE ,
BackgroundJobsTool :: CMD_SERVER ,
[
'cacheServer' ,
$user [ 'id' ],
$serverId
]
2021-05-19 14:55:19 +02:00
);
2021-10-27 17:03:16 +02:00
2021-11-20 14:49:58 +01:00
$this -> out ( " Enqueued cacheServer from $serverName server as job $jobId " );
2021-05-19 14:55:19 +02:00
}
}
2020-02-10 15:15:59 +01:00
public function cacheFeed ()
{
2019-08-08 11:18:21 +02:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ])) {
2019-10-08 09:30:24 +02:00
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Cache feeds for quick lookups' ] . PHP_EOL );
2019-08-08 11:18:21 +02:00
}
2024-04-17 15:08:38 +02:00
2019-08-08 11:18:21 +02:00
$userId = $this -> args [ 0 ];
2021-09-03 08:48:44 +02:00
$user = $this -> getUser ( $userId );
2019-08-08 11:18:21 +02:00
$scope = $this -> args [ 1 ];
if ( ! empty ( $this -> args [ 2 ])) {
$jobId = $this -> args [ 2 ];
} else {
2021-10-29 14:52:42 +02:00
$jobId = $this -> Job -> createJob ( $user , Job :: WORKER_DEFAULT , 'cache_feeds' , 'Feed: ' . $scope , 'Starting feed caching.' );
2019-08-08 11:18:21 +02:00
}
2019-08-28 17:01:07 +02:00
try {
$result = $this -> Feed -> cacheFeedInitiator ( $user , $jobId , $scope );
} catch ( Exception $e ) {
CakeLog :: error ( $e -> getMessage ());
$result = false ;
}
2019-10-02 18:55:37 +02:00
if ( $result === false ) {
$message = __ ( 'Job failed. See error logs for more details.' );
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , false , $message );
2019-10-02 18:55:37 +02:00
2019-08-08 11:18:21 +02:00
} else {
2019-10-02 18:55:37 +02:00
$total = $result [ 'successes' ] + $result [ 'fails' ];
$message = __n (
'%s feed from %s cached. Failed: %s' ,
'%s feeds from %s cached. Failed: %s' ,
2020-09-01 08:42:13 +02:00
$result [ 'successes' ], $result [ 'successes' ], $total , $result [ 'fails' ]
2019-10-02 18:55:37 +02:00
);
if ( $result [ 'fails' ] > 0 ) {
$message .= ' ' . __ ( 'See error logs for more details.' );
}
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , true , $message );
2019-08-08 11:18:21 +02:00
}
echo $message . PHP_EOL ;
}
2016-06-04 01:08:16 +02:00
2020-02-10 15:15:59 +01:00
public function enqueuePull ()
{
2021-03-18 23:44:04 +01:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ]) || empty ( $this -> args [ 2 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Enqueue pull' ] . PHP_EOL );
}
2019-08-08 11:18:21 +02:00
$timestamp = $this -> args [ 0 ];
$userId = $this -> args [ 1 ];
$taskId = $this -> args [ 2 ];
$task = $this -> Task -> read ( null , $taskId );
if ( $timestamp != $task [ 'Task' ][ 'next_execution_time' ]) {
return ;
}
if ( $task [ 'Task' ][ 'timer' ] > 0 ) $this -> Task -> reQueue ( $task , 'default' , 'ServerShell' , 'enqueuePull' , $userId , $taskId );
$user = $this -> User -> getAuthUser ( $userId );
$servers = $this -> Server -> find ( 'all' , array ( 'recursive' => - 1 , 'conditions' => array ( 'pull' => 1 )));
$count = count ( $servers );
$failCount = 0 ;
foreach ( $servers as $k => $server ) {
$this -> Job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'pull' ,
'job_input' => 'Server: ' . $server [ 'Server' ][ 'id' ],
'retries' => 0 ,
'org' => $user [ 'Organisation' ][ 'name' ],
'org_id' => $user [ 'org_id' ],
'process_id' => 'Part of scheduled pull' ,
'message' => 'Pulling.' ,
);
$this -> Job -> save ( $data );
$jobId = $this -> Job -> id ;
2021-09-30 23:19:29 +02:00
$result = $this -> Server -> pull ( $user , 'full' , $server , $jobId );
2019-08-08 11:18:21 +02:00
$this -> Job -> save ( array (
'id' => $jobId ,
'message' => 'Job done.' ,
'progress' => 100 ,
'status' => 4
));
if ( is_numeric ( $result [ 0 ])) {
switch ( $result [ 0 ]) {
case '1' :
$this -> Job -> saveField ( 'message' , 'Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server.' );
break ;
case '2' :
$this -> Job -> saveField ( 'message' , $result [ 1 ]);
break ;
case '3' :
$this -> Job -> saveField ( 'message' , 'Sorry, incremental pushes are not yet implemented.' );
break ;
case '4' :
$this -> Job -> saveField ( 'message' , 'Invalid technique chosen.' );
break ;
2016-06-04 01:08:16 +02:00
2019-08-08 11:18:21 +02:00
}
$failCount ++ ;
}
}
$this -> Task -> id = $task [ 'Task' ][ 'id' ];
$this -> Task -> saveField ( 'message' , count ( $servers ) . ' job(s) completed at ' . date ( 'd/m/Y - H:i:s' ) . '. Failed jobs: ' . $failCount . '/' . $count );
}
2016-06-04 01:08:16 +02:00
2020-02-10 15:15:59 +01:00
public function enqueueFeedFetch ()
{
2021-03-18 23:44:04 +01:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ]) || empty ( $this -> args [ 2 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Enqueue feed fetch' ] . PHP_EOL );
}
2019-08-08 11:18:21 +02:00
$timestamp = $this -> args [ 0 ];
$userId = $this -> args [ 1 ];
$taskId = $this -> args [ 2 ];
$task = $this -> Task -> read ( null , $taskId );
if ( $timestamp != $task [ 'Task' ][ 'next_execution_time' ]) {
return ;
}
if ( $task [ 'Task' ][ 'timer' ] > 0 ) $this -> Task -> reQueue ( $task , 'default' , 'ServerShell' , 'enqueueFeedFetch' , $userId , $taskId );
$user = $this -> User -> getAuthUser ( $userId );
$failCount = 0 ;
2017-09-13 15:35:24 +02:00
$feeds = $this -> Feed -> find ( 'all' , array (
'recursive' => - 1 ,
'conditions' => array ( 'enabled' => true )
));
foreach ( $feeds as $k => $feed ) {
$this -> Job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'feed_fetch' ,
'job_input' => 'Feed: ' . $feed [ 'Feed' ][ 'id' ],
'retries' => 0 ,
'org' => $user [ 'Organisation' ][ 'name' ],
'org_id' => $user [ 'org_id' ],
'process_id' => 'Part of scheduled feed fetch' ,
'message' => 'Pulling.' ,
);
$this -> Job -> save ( $data );
$jobId = $this -> Job -> id ;
$result = $this -> Feed -> downloadFromFeedInitiator ( $feed [ 'Feed' ][ 'id' ], $user , $jobId );
$this -> Job -> save ( array (
'message' => 'Job done.' ,
'progress' => 100 ,
'status' => 4
));
if ( $result !== true ) {
$this -> Job -> saveField ( 'message' , 'Could not fetch feed.' );
$failCount ++ ;
}
}
$this -> Task -> id = $task [ 'Task' ][ 'id' ];
$this -> Task -> saveField ( 'message' , count ( $feeds ) . ' job(s) completed at ' . date ( 'd/m/Y - H:i:s' ) . '. Failed jobs: ' . $failCount . '/' . count ( $feeds ));
2019-08-08 11:18:21 +02:00
}
2017-08-25 14:38:32 +02:00
2020-02-10 15:15:59 +01:00
public function enqueueFeedCache ()
{
2021-03-18 23:44:04 +01:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ]) || empty ( $this -> args [ 2 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Enqueue feed cache' ] . PHP_EOL );
}
2017-09-13 15:35:24 +02:00
$timestamp = $this -> args [ 0 ];
$userId = $this -> args [ 1 ];
$taskId = $this -> args [ 2 ];
$task = $this -> Task -> read ( null , $taskId );
if ( $timestamp != $task [ 'Task' ][ 'next_execution_time' ]) {
return ;
}
2019-08-08 11:18:21 +02:00
if ( $task [ 'Task' ][ 'timer' ] > 0 ) $this -> Task -> reQueue ( $task , 'default' , 'ServerShell' , 'enqueueFeedCache' , $userId , $taskId );
2017-09-13 15:35:24 +02:00
$user = $this -> User -> getAuthUser ( $userId );
$this -> Job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'feed_cache' ,
2018-01-16 23:22:56 +01:00
'job_input' => '' ,
2017-09-13 15:35:24 +02:00
'retries' => 0 ,
'org' => $user [ 'Organisation' ][ 'name' ],
'org_id' => $user [ 'org_id' ],
'process_id' => 'Part of scheduled feed caching' ,
'message' => 'Caching.' ,
);
$this -> Job -> save ( $data );
$jobId = $this -> Job -> id ;
2019-09-02 20:37:18 +02:00
try {
$result = $this -> Feed -> cacheFeedInitiator ( $user , $jobId , 'all' );
} catch ( Exception $e ) {
CakeLog :: error ( $e -> getMessage ());
$result = false ;
}
2019-10-02 18:55:37 +02:00
if ( $result === false ) {
$message = __ ( 'Job failed. See error logs for more details.' );
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , false , $message );
2019-10-02 18:55:37 +02:00
2019-09-02 20:37:18 +02:00
} else {
2019-10-02 18:55:37 +02:00
$total = $result [ 'successes' ] + $result [ 'fails' ];
$message = __n (
'%s feed from %s cached. Failed: %s' ,
'%s feeds from %s cached. Failed: %s' ,
$result [ 'successes' ], $total , $result [ 'fails' ]
);
if ( $result [ 'fails' ] > 0 ) {
$message .= ' ' . __ ( 'See error logs for more details.' );
}
2020-06-26 17:14:18 +02:00
$this -> Job -> saveStatus ( $jobId , true , $message );
2019-09-02 20:37:18 +02:00
}
2017-09-13 15:35:24 +02:00
$this -> Task -> id = $task [ 'Task' ][ 'id' ];
$this -> Task -> saveField ( 'message' , 'Job completed at ' . date ( 'd/m/Y - H:i:s' ));
}
2017-08-25 14:38:32 +02:00
2020-02-10 15:15:59 +01:00
public function enqueuePush ()
{
2021-03-18 23:44:04 +01:00
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ]) || empty ( $this -> args [ 2 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Enqueue push' ] . PHP_EOL );
}
2019-08-08 11:18:21 +02:00
$timestamp = $this -> args [ 0 ];
$taskId = $this -> args [ 1 ];
$userId = $this -> args [ 2 ];
$this -> Task -> id = $taskId ;
$task = $this -> Task -> read ( null , $taskId );
if ( $timestamp != $task [ 'Task' ][ 'next_execution_time' ]) {
return ;
}
if ( $task [ 'Task' ][ 'timer' ] > 0 ) $this -> Task -> reQueue ( $task , 'default' , 'ServerShell' , 'enqueuePush' , $userId , $taskId );
2016-06-04 01:08:16 +02:00
2019-08-08 11:18:21 +02:00
$this -> User -> recursive = - 1 ;
$user = $this -> User -> getAuthUser ( $userId );
$servers = $this -> Server -> find ( 'all' , array ( 'recursive' => - 1 , 'conditions' => array ( 'push' => 1 )));
foreach ( $servers as $k => $server ) {
$this -> Job -> create ();
$data = array (
'worker' => 'default' ,
'job_type' => 'push' ,
'job_input' => 'Server: ' . $server [ 'Server' ][ 'id' ],
'retries' => 0 ,
'org' => $user [ 'Organisation' ][ 'name' ],
'org_id' => $user [ 'org_id' ],
'process_id' => 'Part of scheduled push' ,
'message' => 'Pushing.' ,
);
$this -> Job -> save ( $data );
$jobId = $this -> Job -> id ;
App :: uses ( 'SyncTool' , 'Tools' );
$syncTool = new SyncTool ();
$HttpSocket = $syncTool -> setupHttpSocket ( $server );
2020-06-26 17:14:18 +02:00
$this -> Server -> push ( $server [ 'Server' ][ 'id' ], 'full' , $jobId , $HttpSocket , $user );
2019-08-08 11:18:21 +02:00
}
$this -> Task -> id = $task [ 'Task' ][ 'id' ];
$this -> Task -> saveField ( 'message' , count ( $servers ) . ' job(s) completed at ' . date ( 'd/m/Y - H:i:s' ) . '.' );
}
2021-09-03 08:48:44 +02:00
2022-08-31 11:51:36 +02:00
public function sendPeriodicSummaryToUsers ()
{
2024-04-17 15:08:38 +02:00
2022-08-31 11:51:36 +02:00
$periods = $this -> __getPeriodsForToday ();
$start_time = time ();
echo __n ( 'Started periodic summary generation for the %s period' , 'Started periodic summary generation for periods: %s' , count ( $periods ), implode ( ', ' , $periods )) . PHP_EOL ;
foreach ( $periods as $period ) {
$users = $this -> User -> getSubscribedUsersForPeriod ( $period );
echo __n ( '%s user has subscribed for the `%s` period' , '%s users has subscribed for the `%s` period' , count ( $users ), count ( $users ), $period ) . PHP_EOL ;
foreach ( $users as $user ) {
echo __ ( 'Sending `%s` report to `%s`' , $period , $user [ 'User' ][ 'email' ]) . PHP_EOL ;
$emailTemplate = $this -> User -> generatePeriodicSummary ( $user [ 'User' ][ 'id' ], $period , false );
2022-10-07 12:08:09 +02:00
if ( $emailTemplate === null ) {
continue ; // no new event for this user
}
2022-08-31 11:51:36 +02:00
$this -> User -> sendEmail ( $user , $emailTemplate , false , null );
}
}
2022-09-21 12:47:36 +02:00
echo __ ( 'All reports sent. Task took %s seconds' , time () - $start_time ) . PHP_EOL ;
2022-08-31 11:51:36 +02:00
}
private function __getPeriodsForToday () : array
{
$today = new DateTime ();
$periods = [ 'daily' ];
if ( $today -> format ( 'j' ) == 1 ) {
2022-09-12 15:04:29 +02:00
$periods [] = 'monthly' ;
2022-08-31 11:51:36 +02:00
}
if ( $today -> format ( 'N' ) == 1 ) {
2022-09-12 15:04:29 +02:00
$periods [] = 'weekly' ;
2022-08-31 11:51:36 +02:00
}
return $periods ;
}
2021-09-03 08:48:44 +02:00
/**
* @ param int $userId
* @ return array
*/
private function getUser ( $userId )
{
$user = $this -> User -> getAuthUser ( $userId );
if ( empty ( $user )) {
$this -> error ( 'User ID do not match an existing user.' );
}
return $user ;
}
/**
* @ param int $serverId
* @ return array
*/
private function getServer ( $serverId )
{
$server = $this -> Server -> find ( 'first' , [
'conditions' => [ 'Server.id' => $serverId ],
'recursive' => - 1 ,
]);
if ( ! $server ) {
$this -> error ( " Server with ID $serverId doesn't exists. " );
}
return $server ;
}
2021-11-20 14:49:58 +01:00
2022-01-14 14:54:39 +01:00
public function push_taxii ()
{
if ( empty ( $this -> args [ 0 ]) || empty ( $this -> args [ 1 ])) {
die ( 'Usage: ' . $this -> Server -> command_line_functions [ 'console_automation_tasks' ][ 'data' ][ 'Push Taxii' ] . PHP_EOL );
}
2024-04-17 15:08:38 +02:00
2022-01-14 14:54:39 +01:00
$userId = $this -> args [ 0 ];
$user = $this -> getUser ( $userId );
$serverId = $this -> args [ 1 ];
2023-06-14 18:37:27 +02:00
if ( ! empty ( $this -> args [ 2 ])) {
$jobId = $this -> args [ 2 ];
2022-01-14 14:54:39 +01:00
} else {
$jobId = $this -> Job -> createJob ( $user , Job :: WORKER_DEFAULT , 'push_taxii' , 'Server: ' . $serverId , 'Pushing.' );
}
$this -> Job -> read ( null , $jobId );
2023-06-06 11:12:38 +02:00
$result = $this -> TaxiiServer -> push ( $serverId , $user , $jobId );
2022-01-14 14:54:39 +01:00
if ( $result !== true && ! is_array ( $result )) {
$message = 'Job failed. Reason: ' . $result ;
$this -> Job -> saveStatus ( $jobId , false , $message );
} else {
$message = 'Job done.' ;
$this -> Job -> saveStatus ( $jobId , true , $message );
}
}
2014-01-06 05:15:47 +01:00
}