MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [sync] Use sync tool for pulling proposals

pull/7650/head
Jakub Onderka 2021-08-11 20:47:52 +02:00
parent 1f10a88504
commit 97b16fe552
3 changed files with 61 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app/Lib/Tools/ServerSyncTool.php
View File

@ -6,7 +6,8 @@ class ServerSyncTool
const FEATURE_BR = 'br',
FEATURE_GZIP = 'gzip',
FEATURE_ORG_RULE = 'org_rule',
FEATURE_FILTER_SIGHTINGS = 'filter_sightings';
FEATURE_FILTER_SIGHTINGS = 'filter_sightings',
FEATURE_PROPOSALS = 'proposals';
/** @var array */
private $server;
@ -81,6 +82,19 @@ class ServerSyncTool
return $this->get($url);
}
/**
* @param array $params
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
*/
public function fetchProposals(array $params = [])
{
$url = '/shadow_attributes/index';
$url .= $this->createParams($params);
$url .= '.json';
return $this->get($url);
}
/**
* @param array $event
* @param array $sightingUuids
@ -184,6 +198,9 @@ class ServerSyncTool
case self::FEATURE_ORG_RULE:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] > 123;
case self::FEATURE_PROPOSALS:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] >= 111;
default:
throw new InvalidArgumentException("Invalid flag `$flag` provided");
}

2
app/Model/Server.php
View File

@ -583,7 +583,7 @@ class Server extends AppModel
}
$pulledProposals = $pulledSightings = 0;
if ($technique === 'full' || $technique === 'update') {
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $server);
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
if ($jobId) {
$job->saveProgress($jobId, 'Pulling sightings.', 75);

78
app/Model/ShadowAttribute.php
View File

@ -4,6 +4,7 @@ App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('AttachmentTool', 'Tools');
App::uses('ComplexTypeTool', 'Tools');
App::uses('ServerSyncTool', 'Tools');
/**
* @property Event $Event
@ -602,7 +603,11 @@ class ShadowAttribute extends AppModel
return $proposalCount;
}
private function __preCaptureMassage($proposal)
/**
* @param array $proposal
* @return array|false
*/
private function __preCaptureMassage(array $proposal)
{
if (empty($proposal['event_uuid']) || empty($proposal['Org'])) {
return false;
@ -657,49 +662,50 @@ class ShadowAttribute extends AppModel
return false;
}
public function pullProposals($user, $server, $HttpSocket = null)
/**
* @param array $user
* @param ServerSyncTool $serverSync
* @return int
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function pullProposals(array $user, ServerSyncTool $serverSync)
{
$version = explode('.', $server['Server']['version']);
if (
($version[0] == 2 && $version[1] == 4 && $version[2] < 111)
) {
if (!$serverSync->isSupported(ServerSyncTool::FEATURE_PROPOSALS)) {
return 0;
}
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$i = 1;
$fetchedCount = 0;
$chunk_size = 1000;
$chunkSize = 1000;
$timestamp = strtotime("-90 day");
while(true) {
$uri = sprintf(
'%s/shadow_attributes/index/all:1/timestamp:%s/limit:%s/page:%s/deleted[]:0/deleted[]:1.json',
$url,
$timestamp,
$chunk_size,
$i
);
$i += 1;
$response = $HttpSocket->get($uri, false, $request);
if ($response->code == 200) {
$data = json_decode($response->body, true);
if (empty($data)) {
return $fetchedCount;
}
$returnSize = count($data);
foreach ($data as $k => $proposal) {
$result = $this->capture($proposal['ShadowAttribute'], $user);
if ($result) {
$fetchedCount += 1;
}
}
if ($returnSize < $chunk_size) {
return $fetchedCount;
}
} else {
while (true) {
try {
$data = $serverSync->fetchProposals([
'all' => 1,
'timestamp' => $timestamp,
'limit' => $chunkSize,
'page' => $i,
'deleted' => [0, 1],
])->json();
} catch (Exception $e) {
$this->logException("Could not fetch page $i of proposals from remote server {$serverSync->server()['Server']['id']}", $e);
return $fetchedCount;
}
$returnSize = count($data);
if ($returnSize === 0) {
return $fetchedCount;
}
foreach ($data as $proposal) {
$result = $this->capture($proposal['ShadowAttribute'], $user);
if ($result) {
$fetchedCount++;
}
}
if ($returnSize < $chunkSize) {
return $fetchedCount;
}
$i++;
}
}