chg: [analyst-data:pull] Started implementation of pull - WiP

app/Lib/Tools/ServerSyncTool.php

@@ -224,9 +224,36 @@ class ServerSyncTool
      */
     public function analystDataSearch(array $rules)
     {
+        if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
+            return [];
+        }
+
         return $this->post('/analyst_data/filterAnalystDataForPush', $rules);
     }
 
+    /**
+     * @throws HttpSocketJsonException
+     * @throws HttpSocketHttpException
+     */
+    public function fetchAnalystData(array $uuids)
+    {
+        if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
+            return [];
+        }
+
+        $params = [
+            'uuids' => $uuids,
+        ];
+
+        $url = '/analyst_data/index';
+        $url .= $this->createParams($params);
+        $url .= '.json';
+        return $this->get($url);
+
+        // $response = $this->post('/analyst_data/restSearch' , $params);
+        // return $response->json();
+    }
+
         /**
      * @param array $analystData
      * @return HttpSocketResponseExtended

app/Model/AnalystData.php

@@ -634,4 +634,65 @@ class AnalystData extends AppModel
         }
         return $analystData;
     }
+
+    public function pull(array $user, ServerSyncTool $serverSync)
+    {
+        $this->Server = ClassRegistry::init('Server');
+        $this->AnalystData = ClassRegistry::init('AnalystData');
+        try {
+            $remoteData = $this->Server->fetchAnalystDataIdsFromServer($serverSync);
+        } catch (Exception $e) {
+            $this->logException("Could not fetch analyst data IDs from server {$serverSync->server()['Server']['name']}", $e);
+            return 0;
+        }
+
+        // Downloads new analyst data and the ones newer than local.
+        $localAnalystData = $this->getAllAnalystData('list', [
+            'Event.uuid' => array_column($remoteData, 'uuid')
+        ]);
+
+        $remoteDataUuids = [];
+        foreach ($remoteData as $type => $remoteAnalystData) {
+            foreach ($remoteAnalystData as $i => $remoteEntry) {
+                if (
+                    isset($localAnalystData[$remoteEntry['uuid']]) &&
+                    strtotime($localAnalystData[$type][$remoteEntry['uuid']]) < strtotime($remoteEntry['modified'])
+                ) {
+                    $remoteDataUuids[$remoteEntry['uuid']] = $remoteEntry['modified'];
+                }
+            }
+        }
+        unset($remoteData, $localAnalystData);
+
+        if (empty($remoteDataUuids)) {
+            return 0;
+        }
+
+        if ($serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) {
+            return $this->pullAnalystData($user, $remoteDataUuids, $serverSync);
+        }
+    }
+
+    public function pullAnalystData(array $user, array $analystDataUuids, ServerSyncTool $serverSync)
+    {
+        $uuids = array_keys($analystDataUuids);
+        $saved = 0;
+        foreach (array_chunk($uuids, 100) as $uuidChunk) {
+            try {
+                $chunkedAnalystData = $serverSync->fetchAnalystData($uuidChunk);
+            } catch (Exception $e) {
+                $this->logException("Failed downloading the chunked analyst data from {$serverSync->server()['Server']['name']}.", $e);
+                continue;
+            }
+
+            foreach ($chunkedAnalystData as $analystData) {
+                $savedAmount = $this->captureAnalystData($user, $analystData, true, $serverSync->server()['Server']['org_id'], $serverSync->server());
+                if ($savedAmount) {
+                    $saved += $savedAmount;
+                }
+            }
+        }
+
+        return $saved;
+    }
 }

app/Model/Server.php

@@ -671,21 +671,24 @@ class Server extends AppModel
                 $job->saveProgress($jobId, 'Pulling sightings.', 75);
             }
             $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
+            $this->AnalystData = ClassRegistry::init('AnalystData');
+            $pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
         }
         if ($jobId) {
             $job->saveStatus($jobId, true, 'Pull completed.');
         }
 
         $change = sprintf(
-            '%s events, %s proposals, %s sightings and %s galaxy clusters pulled or updated. %s events failed or didn\'t need an update.',
+            '%s events, %s proposals, %s sightings, %s galaxy clusters and %s pulled or updated. %s events failed or didn\'t need an update.',
             count($successes),
             $pulledProposals,
             $pulledSightings,
             $pulledClusters,
+            $pulledAnalystData,
             count($fails)
         );
         $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change);
-        return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters];
+        return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters, $pulledAnalystData];
     }
 
     public function filterRuleToParameter($filter_rules)
@@ -753,7 +756,7 @@ class Server extends AppModel
      * @return array The list of analyst data
      * @throws JsonException|HttpSocketHttpException|HttpSocketJsonException
      */
-    private function fetchAnalystDataIdsFromServer(ServerSyncTool $serverSync, array $conditions = [])
+    public function fetchAnalystDataIdsFromServer(ServerSyncTool $serverSync, array $conditions = [])
     {
         $filterRules = $conditions;
         $dataArray = $serverSync->analystDataSearch($filterRules)->json();