new: [API] Added rate limiting option to the API

- / role setting - can be enabled/disabled and if enabled a limit can be set - limit counter / 15 minutes starting from the first query - x-headers inform the user about their limit/remaining queries/reset in seconds
2019-10-08 11:43:56 +02:00 · 2019-10-08 11:43:56 +02:00 · a8c57a8316
parent 4c826a74cc
commit a8c57a8316
7 changed files with 110 additions and 12 deletions
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@ -46,7 +46,7 @@ class AppController extends Controller

    public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');

-    private $__queryVersion = '88';
+    private $__queryVersion = '89';
    public $pyMispVersion = '2.4.114';
    public $phpmin = '7.0';
    public $phprec = '7.2';
@ -89,7 +89,8 @@ class AppController extends Controller
            'ACL',
            'RestResponse',
            'Flash',
-            'Toolbox'
+            'Toolbox',
+            'RateLimit'
            //,'DebugKit.Toolbar'
    );

@ -358,7 +359,7 @@ class AppController extends Controller
                if (!$this->User->exists()) {
                    $message = __('Something went wrong. Your user account that you are authenticated with doesn\'t exist anymore.');
                    if ($this->_isRest) {
-                        $this->RestResponse->throwException(
+                        echo $this->RestResponse->throwException(
                            401,
                            $message
                        );
@ -470,6 +471,35 @@ class AppController extends Controller
        }
        $this->set('notifications', $notifications);
        $this->ACL->checkAccess($this->Auth->user(), Inflector::variable($this->request->params['controller']), $this->action);
+        $this->__rateLimitCheck();
+    }
+
+    private function __rateLimitCheck()
+    {
+        $info = array();
+        $rateLimitCheck = $this->RateLimit->check(
+            $this->Auth->user(),
+            $this->request->params['controller'],
+            $this->action,
+            $this->{$this->modelClass},
+            $info,
+            $this->response->type()
+        );
+        if (!empty($info)) {
+            $this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']);
+            $this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']);
+            $this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']);
+        }
+        if ($rateLimitCheck !== true) {
+            $this->response->header('X-Rate-Limit-Limit', $info['limit']);
+            $this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
+            $this->response->header('X-Rate-Limit-Reset', $info['reset']);
+            $this->response->body($rateLimitCheck);
+            $this->response->statusCode(429);
+            $this->response->send();
+            $this->_stop();
+        }
+        return true;
    }

    public function afterFilter()
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@ -4,6 +4,8 @@ class RestResponseComponent extends Component
 {
    public $components = array('ACL');

+    public $headers = array();
+
    private $__convertActionToMessage = array(
        'SharingGroup' => array(
            'addOrg' => 'add Organisation to',
@ -452,13 +454,16 @@ class RestResponseComponent extends Component
            $headers["Access-Control-Allow-Origin"] = explode(',', Configure::read('Security.cors_origins'));
            $headers["Access-Control-Expose-Headers"] = ["X-Result-Count"];
        }
-
+        if (!empty($this->headers)) {
+            foreach ($this->headers as $key => $value) {
+                $cakeResponse->header($key, $value);
+            }
+        }
        if (!empty($headers)) {
            foreach ($headers as $key => $value) {
                $cakeResponse->header($key, $value);
            }
        }
-
        if ($download) {
            $cakeResponse->download($download);
        }
@ -500,14 +505,19 @@ class RestResponseComponent extends Component
        return $cakeResponse;
    }

-    public function throwException($code, $message, $url = '', $format = false, $raw = false)
+    public function throwException($code, $message, $url = '', $format = false, $raw = false, $headers = array())
    {
        $message = array(
            'name' => $message,
            'message' => $message,
            'url' => $url
        );
-        return $this->__sendResponse($message, $code, $format, $raw);
+        return $this->__sendResponse($message, $code, $format, $raw, false, $headers);
+    }
+
+    public function setHeader($header, $value)
+    {
+        $this->headers[$header] = $value;
    }

    public function describe($controller, $action, $id = false, $format = false)
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@ -76,7 +76,7 @@ class AppModel extends Model
        21 => false, 22 => false, 23 => false, 24 => false, 25 => false, 26 => false,
        27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false,
        33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
-        39 => false, 40 => false
+        39 => false, 40 => false, 41 => false
    );

    public $advanced_updates_description = array(
@ -1264,6 +1264,10 @@ class AppModel extends Model
                $sqlArray[] = "ALTER TABLE `user_settings` ADD `timestamp` int(11) NOT NULL;";
                $indexArray[] = array('user_settings', 'timestamp');
                break;
+            case 41:
+                $sqlArray[] = "ALTER TABLE `roles` ADD `enforce_rate_limit` tinyint(1) NOT NULL DEFAULT 0;";
+                $sqlArray[] = "ALTER TABLE `roles` ADD `rate_limit_count` int(11) NOT NULL DEFAULT 0;";
+                break;
            case 'fixNonEmptySharingGroupID':
                $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2417,7 +2421,7 @@ class AppModel extends Model
            }
        }
    }
-    
+
    /**
     * @param string $message
     * @param Exception $exception
--- a/app/View/Roles/admin_add.ctp
+++ b/app/View/Roles/admin_add.ctp
@ -20,6 +20,19 @@
            echo $this->Form->input('max_execution_time', array('label' => __('Maximum execution time') . ' (' . h($default_max_execution_time) . ')'));
        ?>
        <div class = 'input clear'></div>
+        <?php
+            echo $this->Form->input('enforce_rate_limit', array(
+                'type' => 'checkbox',
+                'label' => __('Enforce search rate limit')
+            ));
+        ?>
+        <div class = 'input clear'></div>
+        <div id="rateLimitCountContainer">
+            <?php
+                echo $this->Form->input('rate_limit_count', array('label' => __('# of searches / 15 min')));
+            ?>
+        </div>
+        <div class = 'input clear'></div>
        <?php
            $counter = 1;
            foreach ($permFlags as $k => $flag):
@ -52,8 +65,12 @@ echo $this->Form->end();
 <script type="text/javascript">
    $(document).ready(function() {
        checkRolePerms();
+        checkRoleEnforceRateLimit();
        $(".checkbox, #RolePermission").change(function() {
-        checkRolePerms();
+            checkRolePerms();
+        });
+        $("#RoleEnforceRateLimit").change(function() {
+            checkRoleEnforceRateLimit();
        });
    });
 </script>
--- a/app/View/Roles/admin_edit.ctp
+++ b/app/View/Roles/admin_edit.ctp
@ -19,6 +19,20 @@
            echo $this->Form->input('max_execution_time', array('label' => __('Maximum execution time') . ' (' . h($default_max_execution_time) . ')'));
        ?>
        <div class = 'input clear'></div>
+        <?php
+            echo $this->Form->input('enforce_rate_limit', array(
+                'type' => 'checkbox',
+                'checked' => $this->request->data['Role']['enforce_rate_limit'],
+                'label' => __('Enforce search rate limit')
+            ));
+        ?>
+        <div class = 'input clear'></div>
+        <div id="rateLimitCountContainer">
+            <?php
+                echo $this->Form->input('rate_limit_count', array('label' => __('# of searches / 15 min')));
+            ?>
+        </div>
+        <div class = 'input clear'></div>
        <?php
            $counter = 1;
            foreach ($permFlags as $k => $flag):
@ -50,8 +64,12 @@
 <script type="text/javascript">
    $(document).ready(function() {
        checkRolePerms();
+        checkRoleEnforceRateLimit();
        $(".checkbox, #RolePermission").change(function() {
-        checkRolePerms();
+            checkRolePerms();
+        });
+        $("#RoleEnforceRateLimit").change(function() {
+            checkRoleEnforceRateLimit();
        });
    });
 </script>
--- a/app/View/Roles/admin_index.ctp
+++ b/app/View/Roles/admin_index.ctp
@ -32,6 +32,7 @@
            ?>
            <th><?php echo $this->Paginator->sort('memory_limit', __('Memory limit'));?></th>
            <th><?php echo $this->Paginator->sort('max_execution_time', __('Max execution time'));?></th>
+            <th><?php echo $this->Paginator->sort('rate_limit_count', __('Searches / 15 mins'));?></th>
            <th class="actions"><?php echo __('Actions');?></th>
    </tr><?php
 foreach ($list as $item): ?>
@ -47,7 +48,7 @@ foreach ($list as $item): ?>
                echo sprintf(
                    '<td class="short"><span class="%s" role="img" aria-label="%s" title="%s"></span>&nbsp;</td>',
                    ($item['Role'][$k]) ? 'icon-ok' : '',
-                    ($item['Role'][$k]) ? __('Yes') : __('No'),		    
+                    ($item['Role'][$k]) ? __('Yes') : __('No'),
                    sprintf(
                        __('%s permission %s'),
                        h($flagName),
@ -75,6 +76,15 @@ foreach ($list as $item): ?>
                }
            ?>
        </td>
+        <td class="short">
+            <?php
+                if (empty($item['Role']['rate_limit_count']) || empty($item['Role']['enforce_rate_limit'])) {
+                    echo 'N/A';
+                } else {
+                    echo h(intval($item['Role']['rate_limit_count']));
+                }
+            ?>
+        </td>
        <td class="short action-links">
            <?php echo $this->Html->link('', array('admin' => true, 'action' => 'edit', $item['Role']['id']), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit'))); ?>
            <?php echo $this->Form->postLink('', array('admin' => true, 'action' => 'delete', $item['Role']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete %s?', $item['Role']['name'])); ?>
--- a/app/webroot/js/misp.js
+++ b/app/webroot/js/misp.js
@ -1177,6 +1177,7 @@ function submitPopoverForm(context_id, referer, update_context_id) {
            break;
    }
    if (url !== null) {
+        url = baseurl + url;
        $.ajax({
            beforeSend: function (XMLHttpRequest) {
                $(".loading").show();
@ -4504,6 +4505,14 @@ function moveIndexRow(id, direction, endpoint) {
    });
 }

+function checkRoleEnforceRateLimit() {
+    if ($("#RoleEnforceRateLimit").is(':checked')) {
+        $('#rateLimitCountContainer').show();
+    } else {
+        $('#rateLimitCountContainer').hide();
+    }
+}
+
 (function(){
    "use strict";
    $(".datepicker").datepicker({