From 943e18464293df3dc0cf144fc4ef56c92f617866 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 11 Aug 2021 13:58:12 +0200 Subject: [PATCH 01/27] chg: [app_local] config defaults --- config/app_local.example.php | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/config/app_local.example.php b/config/app_local.example.php index 03aeef1..aca4ef1 100644 --- a/config/app_local.example.php +++ b/config/app_local.example.php @@ -5,6 +5,16 @@ * Note: It is not recommended to commit files with credentials such as app_local.php * into source code version control. */ + +// set the baseurl here if you want to set it manually +$baseurl = env('CEREBRATE_BASEURL', false); + + +// Do not modify the this block +$temp = parse_url($baseurl); +$base = empty($temp['path']) ? false : $temp['path']; +// end of block + return [ /* * Debug Level: @@ -90,8 +100,12 @@ return [ ], ], 'Cerebrate' => [ - 'open' => [], - 'dark' => 0, - 'baseurl' => '' + 'open' => [], + 'dark' => 0, + 'baseurl' => '' + ], + 'App' => [ + 'base' => $base, + 'fullBaseUrl' => $fullBaseUrl ] ]; From 739dc25b1ee88d2d1eae32e4f4432fe41cb0c2d1 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 24 Sep 2021 13:11:39 +0200 Subject: [PATCH 02/27] fix: [Command] typo fixed as mentioned in #71 --- src/Command/ImporterCommand.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index c79ba76..52e11ba 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -2,7 +2,7 @@ /** * Generic importer to feed data to cerebrate from JSON or CSV. - * + * * - JSON configuration file must have the `format` key which can either take the value `json` or `csv` * - If `csv` is provided, the file must contains the header. * - If `json` is provided, a `mapping` key on how to reach each fields using the cakephp4's Hash syntax must be provided. @@ -10,7 +10,7 @@ * - The key is the field name * - The value * - Can either be the string representing the path from which to get the value - * - Or a JSON containg the `path`, the optional `override` parameter specifying if the existing data should be overriden + * - Or a JSON containg the `path`, the optional `override` parameter specifying if the existing data should be overriden * and an optional `massage` function able to alter the data. * - Example * { @@ -22,7 +22,7 @@ * }, * * - The optional primary key argument provides a way to make import replayable. It can typically be used when an ID or UUID is not provided in the source file but can be replaced by something else (e.g. team-name or other type of unique data). - * + * */ namespace App\Command; @@ -161,7 +161,7 @@ class ImporterCommand extends Command 'valueField' => 'id' ])->where(['meta_template_id' => $metaTemplate->id])->toArray(); } else { - $this->io->error("Unkown template for UUID {$config['metaTemplateUUID']}"); + $this->io->error("Unknown template for UUID {$config['metaTemplateUUID']}"); die(1); } } @@ -192,7 +192,7 @@ class ImporterCommand extends Command $metaEntity->meta_template_field_id = $metaTemplateFieldsMapping[$fieldName]; } else { $hasErrors = true; - $this->io->error("Field $fieldName is unkown for template {$metaTemplate->name}"); + $this->io->error("Field $fieldName is unknown for template {$metaTemplate->name}"); break; } } @@ -525,4 +525,4 @@ class ImporterCommand extends Command { return is_null($value) ? '' : $value; } -} \ No newline at end of file +} From 7eac09a3023f4b1884b1c9afc914695b69a0ca23 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 22 Oct 2021 16:25:47 +0200 Subject: [PATCH 03/27] chg: [doc] README improved for release 1.0 --- README.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c2dfa33..33d50fa 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,31 @@ # cerebrate -The Cerebrate Sync Platform core software. Cerebrate is an open-source platform meant to act as a trusted contact information provider and interconnection orchestrator for other security tools. +Cerebrate is an [open-source platform](https://github.com/cerebrate-project) meant to act as a trusted contact information provider and interconnection orchestrator for other security tools (such as [MISP](https://www.misp-project.org/)). -It is currently being built under the MeliCERTes v2 project and is heavily work in progress. +# Features -# Current features +- Advanced repository to manage individuals and organisations; +- Key store for public encryption and signing cryptographic keys (e.g. PGP); +- Distributed synchronisation model where multiple Cerebrate instances can be interconnected amongst organisations and/or departments; +- Management of individuals and their affiliations to each organisations; +- Advanced API and CLI to integrate with existing tools (e.g. importing existing directory information); +- Dynamic model for creating new organisational structure; +- Support existing organisational structures such as [FIRST.org](https://www.first.org/) directory, EU [CSIRTs network](https://csirtsnetwork.eu/); +- Local tooling interconnection to easily interconnect existing tools with their native protocols; -- Repository of organisations and individuals -- Maintain signing and encryption keys -- Maintain affiliations between organisations and individuals +Cerebrate is developed in the scope of the MeliCERTes v2 project. ## Screenshots +![Dashboard](https://www.cerebrate-project.org/assets/images/screenshots/Screenshot%20from%202021-10-19%2016-31-56.png) + List of individuals along with their affiliations -![List of individuals](/documentation/images/individuals.png) +![List of individuals](https://www.cerebrate-project.org/assets/images/screenshots/Screenshot%20from%202021-10-19%2016-32-35.png) Adding organisations -![Adding an organisation](/documentation/images/add_org.png) +![Adding an organisation](https://www.cerebrate-project.org/assets/images/screenshots/Screenshot%20from%202021-10-19%2016-33-04.png) Everything is available via the API, here an example of a search query for all international organisations in the DB. @@ -28,6 +35,10 @@ Managing public keys and assigning them to users both for communication and vali ![Encryption key management](/documentation/images/add_encryption_key.png) +Dynamic model for creating new organisation structre + +![Meta Field Templates](https://www.cerebrate-project.org/assets/images/screenshots/Screenshot%20from%202021-10-19%2016-38-21.png) + # Requirements and installation The platform is built on CakePHP 4 along with Bootstrap 4 and shares parts of the code-base with [MISP](https://www.github.com/MISP). @@ -45,6 +56,7 @@ For installation via docker, refer to the [cerebrate-docker](https://github.com/ ~~~~ The software is released under the AGPLv3. - Copyright (C) 2019, 2020 Andras Iklody + Copyright (C) 2019, 2021 Andras Iklody + Copyright (C) 2020-2021 Sami Mokaddem Copyright (C) CIRCL - Computer Incident Response Center Luxembourg ~~~~ From 3050f7e23b9d3939020116b6121d6bdd9108a75f Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Fri, 22 Oct 2021 16:44:11 +0200 Subject: [PATCH 04/27] chg: small changes to the readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33d50fa..0114d21 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ Cerebrate is an [open-source platform](https://github.com/cerebrate-project) mea - Distributed synchronisation model where multiple Cerebrate instances can be interconnected amongst organisations and/or departments; - Management of individuals and their affiliations to each organisations; - Advanced API and CLI to integrate with existing tools (e.g. importing existing directory information); -- Dynamic model for creating new organisational structure; +- Dynamic model for creating new organisational structures; - Support existing organisational structures such as [FIRST.org](https://www.first.org/) directory, EU [CSIRTs network](https://csirtsnetwork.eu/); -- Local tooling interconnection to easily interconnect existing tools with their native protocols; +- Local tooling interconnection to easily connect existing tools with their native protocols; Cerebrate is developed in the scope of the MeliCERTes v2 project. From 058314af52614fae57c0bf10c2cb29ad0ea77042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Laurent?= <33688507+remil1000@users.noreply.github.com> Date: Mon, 25 Oct 2021 15:46:04 +0200 Subject: [PATCH 05/27] Create docker-publish.yml initial attempt at GH actions docker build and push --- .github/workflows/docker-publish.yml | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..c08958f --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,52 @@ +name: Docker + +on: + push: + branches: [ feature/docker-ci ] + tags: [ 'v*.*' ] + pull_request: + branches: [ main ] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # https://github.com/docker/build-push-action + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + file: docker/Dockerfile + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + "COMPOSER_VERSION=2.1.5" + "PHP_VERSION=7.4" + "DEBIAN_RELEASE=buster" From cfe8b7cd46aa7410859be93aa153f17e5aa8d378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Laurent?= Date: Mon, 25 Oct 2021 15:46:47 +0200 Subject: [PATCH 06/27] dockerfile and configuration --- .gitignore | 2 +- docker/Dockerfile | 80 ++++++++++++++++++++++++++++++++ docker/README.md | 47 +++++++++++++++++++ docker/docker-compose.yml | 28 +++++++++++ docker/entrypoint.sh | 20 ++++++++ docker/etc/DocumentRoot.htaccess | 3 ++ docker/etc/app_local.php | 44 ++++++++++++++++++ docker/etc/webroot.htaccess | 3 ++ 8 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yml create mode 100755 docker/entrypoint.sh create mode 100644 docker/etc/DocumentRoot.htaccess create mode 100644 docker/etc/app_local.php create mode 100644 docker/etc/webroot.htaccess diff --git a/.gitignore b/.gitignore index f370d3e..cdb86ef 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ config/app_local.php logs tmp vendor - +docker/run/ diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..7e2035e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,80 @@ +ARG COMPOSER_VERSION +ARG PHP_VERSION +ARG DEBIAN_RELEASE + +FROM php:${PHP_VERSION}-apache-${DEBIAN_RELEASE} + +# we need some extra libs to be installed in the runtime +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl git zip unzip && \ + apt-get install -y --no-install-recommends libicu-dev libxml2-dev && \ + docker-php-ext-install intl pdo pdo_mysql mysqli simplexml && \ + apt-get remove -y --purge libicu-dev libxml2-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY composer.json composer.json + +# install composer as root +ARG COMPOSER_VERSION +RUN curl -sL https://getcomposer.org/installer | \ + php -- --install-dir=/usr/bin/ --filename=composer --version=${COMPOSER_VERSION} + +# switch back to unprivileged user for composer install +USER www-data + +RUN composer install \ + --ignore-platform-reqs \ + --no-interaction \ + --no-plugins \ + --no-scripts \ + --prefer-dist + +# web server configuration +USER root + +# allow .htaccess overrides and push them +RUN a2enmod rewrite +RUN sed -i -r '/DocumentRoot/a \\t\n\t\tAllowOverride all\n\t' /etc/apache2/sites-available/000-default.conf +COPY --chown=www-data docker/etc/DocumentRoot.htaccess /var/www/html/.htaccess +COPY --chown=www-data docker/etc/webroot.htaccess /var/www/html/webroot/.htaccess + +# passing environment variables through apache +RUN a2enmod env +RUN echo 'PassEnv CEREBRATE_DB_HOST' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_DB_NAME' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_DB_PASSWORD' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_DB_PORT' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_DB_SCHEMA' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_DB_USERNAME' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_EMAIL_HOST' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_EMAIL_PASSWORD' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_EMAIL_PORT' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_EMAIL_TLS' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_EMAIL_USERNAME' >> /etc/apache2/conf-enabled/environment.conf +RUN echo 'PassEnv CEREBRATE_SECURITY_SALT' >> /etc/apache2/conf-enabled/environment.conf + +# entrypoint +COPY docker/entrypoint.sh /entrypoint.sh +RUN chmod 755 /entrypoint.sh + +# copy actual codebase +COPY --chown=www-data . /var/www/html + +# last checks with unprivileged user +USER www-data + +# CakePHP seems to not handle very well externally installed components +# this will chown/chmod/symlink all in place for its own good +RUN composer install --no-interaction + +# app config override making use of environment variables +COPY --chown=www-data docker/etc/app_local.php /var/www/html/config/app_local.php +# version 1.0 addition requires a config/config.json file +# can still be overriden by a docker volume +RUN cp -a /var/www/html/config/config.example.json /var/www/html/config/config.json + +# also can be overridin by a docker volume +RUN mkdir -p /var/www/html/logs + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..1074c0c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,47 @@ +# Database init + +For the `docker-compose` setup to work you must initialize database with +what is in `../INSTALL/mysql.sql` + +``` +mkdir -p run/dbinit/ +cp ../INSTALL/mysql.sql run/dbinit/ +``` + +The MariaDB container has a volume mounted as follow +`- ./run/dbinit:/docker-entrypoint-initdb.d/:ro` + +So that on startup the container will source files in this directory to seed +the database. Once it's done the container will run normally and Cerebrate will +be able to roll its database migration scripts + +# Actual data and volumes + +The actual database will be located in `./run/database` exposed with the +following volume `- ./run/database:/var/lib/mysql` + +Application logs (CakePHP / Cerebrate) will be stored in `./run/logs`, +volume `- ./run/logs:/var/www/html/logs` + +You're free to change those parameters if you're using Swarm, Kubernetes or +your favorite config management tool to deploy this stack + +# Building yourself + +You can create the following Makefile in basedir of this repository +and issue `make image` + +``` +COMPOSER_VERSION?=2.1.5 +PHP_VERSION?=7.4 +DEBIAN_RELEASE?=buster +IMAGE_NAME?=cerebrate:latest + +image: + docker build -t $(IMAGE_NAME) \ + -f docker/Dockerfile \ + --build-arg COMPOSER_VERSION=$(COMPOSER_VERSION) \ + --build-arg PHP_VERSION=$(PHP_VERSION) \ + --build-arg DEBIAN_RELEASE=$(DEBIAN_RELEASE) \ + . +``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..e36c8c5 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3" +services: + database: + image: mariadb:10.6 + restart: always + volumes: + - ./run/database:/var/lib/mysql + - ./run/dbinit:/docker-entrypoint-initdb.d/:ro + environment: + MARIADB_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: "cerebrate" + MYSQL_USER: "cerebrate" + MYSQL_PASSWORD: "etarberec" + www: + image: $IMAGE_NAME + ports: + - "8080:80" + volumes: + - ./run/logs:/var/www/html/logs + environment: + DEBUG: "true" + CEREBRATE_DB_USERNAME: "cerebrate" + CEREBRATE_DB_PASSWORD: "etarberec" + CEREBRATE_DB_NAME: "cerebrate" + CEREBRATE_DB_HOST: database + CEREBRATE_SECURITY_SALT: supersecret + depends_on: + - database diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..35ef6dd --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e + + +run_all_migrations() { + ./bin/cake migrations migrate + ./bin/cake migrations migrate -p tags + ./bin/cake migrations migrate -p ADmad/SocialAuth +} + +# waiting for DB to come up +for try in 1 2 3 4 5 6; do + echo >&2 "migration - attempt $try" + run_all_migrations && break || true + sleep 5 + [ "$try" = "6" ] && exit 1 +done + +exec /usr/local/bin/apache2-foreground "$@" diff --git a/docker/etc/DocumentRoot.htaccess b/docker/etc/DocumentRoot.htaccess new file mode 100644 index 0000000..ef9940b --- /dev/null +++ b/docker/etc/DocumentRoot.htaccess @@ -0,0 +1,3 @@ +RewriteEngine on +RewriteRule ^$ webroot/ [L] +RewriteRule (.*) webroot/$1 [L] diff --git a/docker/etc/app_local.php b/docker/etc/app_local.php new file mode 100644 index 0000000..ac1d212 --- /dev/null +++ b/docker/etc/app_local.php @@ -0,0 +1,44 @@ + env('CEREBRATE_DB_USERNAME', 'cerebrate'), + 'password' => env('CEREBRATE_DB_PASSWORD', ''), + 'host' => env('CEREBRATE_DB_HOST', 'localhost'), + 'database' => env('CEREBRATE_DB_NAME', 'cerebrate'), +]; + +// non-default port can be set on demand - otherwise the DB driver will choose the default +if (!empty(env('CEREBRATE_DB_PORT'))) { + $db['port'] = env('CEREBRATE_DB_PORT'); +} + +// If not using the default 'public' schema with the PostgreSQL driver set it here. +if (!empty(env('CEREBRATE_DB_SCHEMA'))) { + $db['schema'] = env('CEREBRATE_DB_SCHEMA'); +} + +return [ + 'debug' => filter_var(env('DEBUG', false), FILTER_VALIDATE_BOOLEAN), + + 'Security' => [ + 'salt' => env('CEREBRATE_SECURITY_SALT'), + ], + + 'Datasources' => [ + 'default' => $db, + ], + + 'EmailTransport' => [ + 'default' => [ + // host could be ssl://smtp.gmail.com then set port to 465 + 'host' => env('CEREBRATE_EMAIL_HOST', 'localhost'), + 'port' => env('CEREBRATE_EMAIL_PORT', 25), + 'username' => env('CEREBRATE_EMAIL_USERNAME', null), + 'password' => env('CEREBRATE_EMAIL_PASSWORD', null), + 'tls' => env('CEREBRATE_EMAIL_TLS', null) + ], + ], + 'Cerebrate' => [ + 'open' => [], + 'dark' => 0 + ] +]; diff --git a/docker/etc/webroot.htaccess b/docker/etc/webroot.htaccess new file mode 100644 index 0000000..879f805 --- /dev/null +++ b/docker/etc/webroot.htaccess @@ -0,0 +1,3 @@ +RewriteEngine On +RewriteCond %{REQUEST_FILENAME} !-f +RewriteRule ^ index.php [L] From 4dc79a2a5137853384f945689ae0231eae1c33f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Laurent?= Date: Mon, 25 Oct 2021 16:31:06 +0200 Subject: [PATCH 07/27] [skip ci] changing triggering branch in workflow --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c08958f..dd01dc8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,7 +2,7 @@ name: Docker on: push: - branches: [ feature/docker-ci ] + branches: [ main ] tags: [ 'v*.*' ] pull_request: branches: [ main ] From 3916941e07f8f18badf65365ff83885cb29ed7c3 Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Mon, 25 Oct 2021 18:00:33 +0200 Subject: [PATCH 08/27] chg: [docker] updated image path to the github package of this repo --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e36c8c5..c99ee69 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,7 +12,7 @@ services: MYSQL_USER: "cerebrate" MYSQL_PASSWORD: "etarberec" www: - image: $IMAGE_NAME + image: ghcr.io/cerebrate-project/cerebrate:main ports: - "8080:80" volumes: From 8df97082584fc87a1e103fc8df399681eb47f434 Mon Sep 17 00:00:00 2001 From: DocArmoryTech Date: Thu, 28 Oct 2021 22:23:38 +0100 Subject: [PATCH 09/27] Added missing 'Cerebrate' section Second part of resolution to Issue #75 Added missing Cerebrate config section to resolved the following error: ``` warning: Warning (2): in_array() expects parameter 2 to be array, null given in [/var/www/cerebrate/src/Controller/Component/Navigation/sidemenu.php, line 130] Request URL: /users/login Referer URL: http://127.0.0.1:8000/users/login?redirect=%2F Client IP: 127.0.0.1 ``` --- config/app_local.example.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/app_local.example.php b/config/app_local.example.php index 87b1568..1ec0f4a 100644 --- a/config/app_local.example.php +++ b/config/app_local.example.php @@ -99,4 +99,8 @@ return [ 'url' => env('EMAIL_TRANSPORT_DEFAULT_URL', null), ], ], + 'Cerebrate' => [ + 'open' => [], + 'dark' => 0 + ] ]; From 27c2d07e3ccdf552d728d4a85561a92432825a12 Mon Sep 17 00:00:00 2001 From: DocArmoryTech Date: Thu, 28 Oct 2021 22:56:25 +0100 Subject: [PATCH 10/27] Keep composer happy with permissions partial resolution to issue #75 create/initialise a `/var/www/.composer` director to keep composer happy and explicitly tell sudo to set the home dir `-H` --- INSTALL/INSTALL.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/INSTALL/INSTALL.md b/INSTALL/INSTALL.md index a962de1..49948e8 100644 --- a/INSTALL/INSTALL.md +++ b/INSTALL/INSTALL.md @@ -32,8 +32,10 @@ sudo -u www-data git clone https://github.com/cerebrate-project/cerebrate.git /v Run composer ```bash +mkdir -p /var/www/.composer +sudo chown www-data:www-data /var/www/.composer cd /var/www/cerebrate -sudo -u www-data composer install +sudo -H -u www-data composer install ``` Create a database for cerebrate From f10e0225634ece299a4a52b9ace2f6292b55c853 Mon Sep 17 00:00:00 2001 From: DocArmoryTech Date: Thu, 28 Oct 2021 22:58:38 +0100 Subject: [PATCH 11/27] Create logs dir --- logs/.gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 logs/.gitkeep diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/logs/.gitkeep @@ -0,0 +1 @@ + From 5c48de77795acfcdaca1b43d838a46e6db9e0159 Mon Sep 17 00:00:00 2001 From: drizzit56 <60116496+drizzit56@users.noreply.github.com> Date: Fri, 29 Oct 2021 00:29:46 +0100 Subject: [PATCH 12/27] Adding nginx alternative config file and updating INSTALL.md for nginx usage --- INSTALL/INSTALL.md | 36 +++++++++++++++---- ...ate_dev.conf => cerebrate_apache_dev.conf} | 0 INSTALL/cerebrate_nginx.conf | 0 3 files changed, 30 insertions(+), 6 deletions(-) rename INSTALL/{cerebrate_dev.conf => cerebrate_apache_dev.conf} (100%) mode change 100755 => 100644 create mode 100644 INSTALL/cerebrate_nginx.conf diff --git a/INSTALL/INSTALL.md b/INSTALL/INSTALL.md index 49948e8..3685b00 100644 --- a/INSTALL/INSTALL.md +++ b/INSTALL/INSTALL.md @@ -1,8 +1,9 @@ ## Requirements An Ubuntu server (18.04/20.04 should both work fine) - though other linux installations should work too. -- apache2, mysql/mariadb, sqlite need to be installed and running +- apache2 (or nginx), mysql/mariadb, sqlite need to be installed and running - php extensions for intl, mysql, sqlite3, mbstring, xml need to be installed and running +- php extention for curl (not required but makes composer run a little faster) - composer ## Network requirements @@ -17,8 +18,16 @@ Cerebrate communicates via HTTPS so in order to be able to connect to other cere ## Cerebrate installation instructions It should be sufficient to issue the following command to install the dependencies: + +- for apache + ```bash -sudo apt install apache2 mariadb-server git composer php-intl php-mbstring php-dom php-xml unzip php-ldap php-sqlite3 sqlite libapache2-mod-php php-mysql +sudo apt install apache2 mariadb-server git composer php-intl php-mbstring php-dom php-xml unzip php-ldap php-sqlite3 php-curl sqlite libapache2-mod-php php-mysql +``` + +- for nginx +```bash +sudo apt install nginx mariadb-server git composer php-intl php-mbstring php-dom php-xml unzip php-ldap php-sqlite3 sqlite php-fpm php-curl php-mysql ``` Clone this repository (for example into /var/www/cerebrate) @@ -73,7 +82,7 @@ sudo -u www-data cp -a /var/www/cerebrate/config/config.example.json /var/www/ce sudo -u www-data vim /var/www/cerebrate/config/app_local.php ``` -mod_rewrite needs to be enabled: +mod_rewrite needs to be enabled if __using apache__: ```bash sudo a2enmod rewrite @@ -106,16 +115,31 @@ sudo rm /var/www/cerebrate/tmp/cache/persistent/* Create an apache config file for cerebrate / ssh key and point the document root to /var/www/cerebrate/webroot and you're good to go -For development installs the following can be done: +For development installs the following can be done for either apache or nginx: ```bash +# Apache # This configuration is purely meant for local installations for development / testing # Using HTTP on an unhardened apache is by no means meant to be used in any production environment -sudo cp /var/www/cerebrate/INSTALL/cerebrate_dev.conf /etc/apache2/sites-available/ -sudo ln -s /etc/apache2/sites-available/cerebrate_dev.conf /etc/apache2/sites-enabled/ +sudo cp /var/www/cerebrate/INSTALL/cerebrate_apache_dev.conf /etc/apache2/sites-available/ +sudo ln -s /etc/apache2/sites-available/cerebrate_apache_dev.conf /etc/apache2/sites-enabled/ sudo service apache2 restart ``` +OR + +```bash +# NGINX +# This configuration is purely meant for local installations for development / testing +# Using HTTP on an unhardened apache is by no means meant to be used in any production environment +sudo cp /var/www/cerebrate/INSTALL/cerebrate_nginx.conf /etc/nginx/sites-available/ +sudo ln -s /etc/nginx/sites-available/cerebrate_nginx.conf /etc/nginx/sites-enabled/ +sudo systemctl disable apache2 # may be required if apache is using port +sudo service nginx restart +sudo systemctl enable nginx + +``` + Now you can point your browser to: http://localhost:8000 To log in use the default credentials below: diff --git a/INSTALL/cerebrate_dev.conf b/INSTALL/cerebrate_apache_dev.conf old mode 100755 new mode 100644 similarity index 100% rename from INSTALL/cerebrate_dev.conf rename to INSTALL/cerebrate_apache_dev.conf diff --git a/INSTALL/cerebrate_nginx.conf b/INSTALL/cerebrate_nginx.conf new file mode 100644 index 0000000..e69de29 From e3168dacfad7980c9012f2a20e748231c21ce465 Mon Sep 17 00:00:00 2001 From: drizzit56 <60116496+drizzit56@users.noreply.github.com> Date: Fri, 29 Oct 2021 00:33:11 +0100 Subject: [PATCH 13/27] adding nginx config --- INSTALL/cerebrate_nginx.conf | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/INSTALL/cerebrate_nginx.conf b/INSTALL/cerebrate_nginx.conf index e69de29..59ad861 100644 --- a/INSTALL/cerebrate_nginx.conf +++ b/INSTALL/cerebrate_nginx.conf @@ -0,0 +1,37 @@ +## Cerebrate Nginx Web Server Configuration +server { + listen 8000; + # listen 443 ssl; + + root /var/www/cerebrate/webroot; + error_log /var/log/nginx/cerebrate_error.log; + access_log /var/log/nginx/cerebrate_access.log; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html index.php; + + server_name _; + + # Configure Crypto Keys/Certificates/DH + # If enabling this setting change port above, should also set the server name + # ssl_certificate /path/to/ssl/cert; + # ssl_certificate_key /path/to/ssl/cert; + + # enable HSTS + # add_header Strict-Transport-Security "max-age=15768000; includeSubdomains"; + # add_header X-Frame-Options SAMEORIGIN; + + location / { + try_files $uri $uri/ /index.php?$args; + } + + location ~ \.php$ { + try_files $uri =404; + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_pass unix:/var/run/php/php7.4-fpm.sock; + fastcgi_index index.php; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + } +} From ae4a01160036bb1f31795d051b1aac5e6cdf1a8a Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Mon, 1 Nov 2021 16:58:10 +0100 Subject: [PATCH 14/27] Update INSTALL.md Minor installation documentation changes --- INSTALL/INSTALL.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/INSTALL/INSTALL.md b/INSTALL/INSTALL.md index 49948e8..f09ae73 100644 --- a/INSTALL/INSTALL.md +++ b/INSTALL/INSTALL.md @@ -32,7 +32,7 @@ sudo -u www-data git clone https://github.com/cerebrate-project/cerebrate.git /v Run composer ```bash -mkdir -p /var/www/.composer +sudo mkdir -p /var/www/.composer sudo chown www-data:www-data /var/www/.composer cd /var/www/cerebrate sudo -H -u www-data composer install @@ -40,6 +40,11 @@ sudo -H -u www-data composer install Create a database for cerebrate +With a fresh install of Ubuntu sudo to the (system) root user before logging in as the mysql root +```Bash +sudo -i mysql -u root +``` + From SQL shell: ```mysql mysql @@ -48,6 +53,7 @@ CREATE USER 'cerebrate'@'localhost' IDENTIFIED BY 'YOUR_PASSWORD'; GRANT USAGE ON *.* to cerebrate@localhost; GRANT ALL PRIVILEGES ON cerebrate.* to cerebrate@localhost; FLUSH PRIVILEGES; +QUIT; ``` Or from Bash: @@ -93,9 +99,9 @@ This would be, when following the steps above: Run the database schema migrations ```bash -/var/www/cerebrate/bin/cake migrations migrate -/var/www/cerebrate/bin/cake migrations migrate -p tags -/var/www/cerebrate/bin/cake migrations migrate -p ADmad/SocialAuth +sudo -u www-data /var/www/cerebrate/bin/cake migrations migrate +sudo -u www-data /var/www/cerebrate/bin/cake migrations migrate -p tags +sudo -u www-data /var/www/cerebrate/bin/cake migrations migrate -p ADmad/SocialAuth ``` Clean cakephp caches From 23dc460359b980becc23be813619a200ade7a822 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 14:44:07 +0100 Subject: [PATCH 15/27] new: [auditlog system] added - port of Jakub Onderka's implementation from MISP - Still not fully realised, lacking search functionalities --- src/Controller/AuditLogsController.php | 36 ++++ src/Model/Behavior/AuditLogBehavior.php | 214 ++++++++++++++++++++ src/Model/Entity/AuditLog.php | 68 +++++++ src/Model/Table/AuditLogsTable.php | 250 ++++++++++++++++++++++++ 4 files changed, 568 insertions(+) create mode 100644 src/Controller/AuditLogsController.php create mode 100644 src/Model/Behavior/AuditLogBehavior.php create mode 100644 src/Model/Entity/AuditLog.php create mode 100644 src/Model/Table/AuditLogsTable.php diff --git a/src/Controller/AuditLogsController.php b/src/Controller/AuditLogsController.php new file mode 100644 index 0000000..58a2133 --- /dev/null +++ b/src/Controller/AuditLogsController.php @@ -0,0 +1,36 @@ +CRUD->index([ + 'contain' => $this->containFields, + 'filters' => $this->filterFields, + 'quickFilters' => $this->quickFilterFields, + 'afterFind' => function($data) { + $data['request_ip'] = inet_ntop(stream_get_contents($data['request_ip'])); + $data['change'] = stream_get_contents($data['change']); + return $data; + } + ]); + $responsePayload = $this->CRUD->getResponsePayload(); + if (!empty($responsePayload)) { + return $responsePayload; + } + $this->set('metaGroup', 'Administration'); + } +} diff --git a/src/Model/Behavior/AuditLogBehavior.php b/src/Model/Behavior/AuditLogBehavior.php new file mode 100644 index 0000000..3eacfe2 --- /dev/null +++ b/src/Model/Behavior/AuditLogBehavior.php @@ -0,0 +1,214 @@ + true, + 'lastpushedid' => true, + 'timestamp' => true, + 'revision' => true, + 'modified' => true, + 'date_modified' => true, // User + 'current_login' => true, // User + 'last_login' => true, // User + 'newsread' => true, // User + 'proposal_email_lock' => true, // Event + ]; + + public function initialize(array $config): void + { + $this->config = $config; + } + + public function beforeSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $fields = $entity->extract($entity->getVisible(), true); + $skipFields = $this->skipFields; + $fieldsToFetch = array_filter($fields, function($key) use ($skipFields) { + return strpos($key, '_') !== 0 && !isset($skipFields[$key]); + }, ARRAY_FILTER_USE_KEY); + // Do not fetch old version when just few fields will be fetched + $fieldToFetch = []; + if (!empty($options['fieldList'])) { + foreach ($options['fieldList'] as $field) { + if (!isset($this->skipFields[$field])) { + $fieldToFetch[] = $field; + } + } + if (empty($fieldToFetch)) { + $this->old = null; + return true; + } + } + if ($entity->id) { + $this->old = $this->_table->find()->where(['id' => $entity->id])->contain($fieldToFetch)->first(); + } else { + $this->old = null; + } + return true; + } + + public function afterSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + if ($entity->id) { + $id = $entity->id; + } else { + $id = null; + } + + if ($entity->isNew()) { + $action = $entity->getConstant('ACTION_ADD'); + } else { + $action = $entity->getConstant('ACTION_EDIT'); + if (isset($entity['deleted'])) { + if ($entity['deleted']) { + $action = $entity->getConstant('ACTION_SOFT_DELETE'); + } else if (!$entity['deleted'] && $this->old['deleted']) { + $action = $entity->getConstant('ACTION_UNDELETE'); + } + } + } + $changedFields = $this->changedFields($entity, isset($options['fieldList']) ? $options['fieldList'] : null); + if (empty($changedFields)) { + return; + } + + $modelTitleField = $this->_table->getDisplayField(); + if (is_callable($modelTitleField)) { + $modelTitle = $modelTitleField($entity, isset($this->old) ? $this->old : []); + } else if (isset($entity[$modelTitleField])) { + $modelTitle = $entity[$modelTitleField]; + } else if ($this->old[$modelTitleField]) { + $modelTitle = $this->old[$modelTitleField]; + } + $this->auditLogs()->insert([ + 'action' => $action, + 'model' => $entity->getSource(), + 'model_id' => $id, + 'model_title' => $modelTitle, + 'change' => $changedFields + ]); + } + + public function beforeDelete(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $this->old = $this->_table->find()->where(['id' => $entity->id])->first(); + return true; + } + + public function afterDelete(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $modelTitleField = $this->_table->getDisplayField(); + if (is_callable($modelTitleField)) { + $modelTitle = $modelTitleField($entity, []); + } else if (isset($entity[$modelTitleField])) { + $modelTitle = $entity[$modelTitleField]; + } + + $logEntity = $this->auditLogs()->newEntity([ + 'action' => $entity->getConstant('ACTION_DELETE'), + 'model' => $entity->getSource(), + 'model_id' => $this->old->id, + 'model_title' => $modelTitle, + 'change' => $this->changedFields($entity) + ]); + $logEntity->save(); + } + + /** + * @param Model $model + * @param array|null $fieldsToSave + * @return array + */ + private function changedFields(EntityInterface $entity, $fieldsToSave = null) + { + $dbFields = $this->_table->getSchema()->typeMap(); + $changedFields = []; + foreach ($entity->extract($entity->getVisible()) as $key => $value) { + if (isset($this->skipFields[$key])) { + continue; + } + if (!isset($dbFields[$key])) { + continue; + } + if ($fieldsToSave && !in_array($key, $fieldsToSave, true)) { + continue; + } + if (isset($entity[$key]) && isset($this->old[$key])) { + $old = $this->old[$key]; + } else { + $old = null; + } + // Normalize + if (is_bool($old)) { + $old = $old ? 1 : 0; + } + if (is_bool($value)) { + $value = $value ? 1 : 0; + } + $dbType = $dbFields[$key]; + if ($dbType === 'integer' || $dbType === 'tinyinteger' || $dbType === 'biginteger' || $dbType === 'boolean') { + $value = (int)$value; + if ($old !== null) { + $old = (int)$old; + } + } + if ($value == $old) { + continue; + } + if ($key === 'password' || $key === 'authkey') { + $value = '*****'; + if ($old !== null) { + $old = $value; + } + } + + if ($old === null) { + $changedFields[$key] = $value; + } else { + $changedFields[$key] = [$old, $value]; + } + } + return $changedFields; + } + + /** + * @return AuditLogs + */ + public function auditLogs() + { + if ($this->AuditLogs === null) { + $this->AuditLogs = TableRegistry::getTableLocator()->get('AuditLogs'); + } + return $this->AuditLogs; + } + + public function log() + { + + } +} diff --git a/src/Model/Entity/AuditLog.php b/src/Model/Entity/AuditLog.php new file mode 100644 index 0000000..beabeb8 --- /dev/null +++ b/src/Model/Entity/AuditLog.php @@ -0,0 +1,68 @@ +compressionEnabled = Configure::read('Cerebrate.log_compress') && function_exists('brotli_compress'); + parent::__construct($properties, $options); + } + + protected function _getTitle(): String + { + return $this->generateUserFriendlyTitle($this); + } + + /** + * @param string $change + * @return array|string + * @throws JsonException + */ + private function decodeChange($change) + { + if (substr($change, 0, 4) === self::BROTLI_HEADER) { + if (function_exists('brotli_uncompress')) { + $change = brotli_uncompress(substr($change, 4)); + if ($change === false) { + return 'Compressed'; + } + } else { + return 'Compressed'; + } + } + return json_decode($change, true); + } + + /** + * @param array $auditLog + * @return string + */ + public function generateUserFriendlyTitle($auditLog) + { + if (in_array($auditLog['action'], [self::ACTION_TAG, self::ACTION_TAG_LOCAL, self::ACTION_REMOVE_TAG, self::ACTION_REMOVE_TAG_LOCAL], true)) { + $attached = ($auditLog['action'] === self::ACTION_TAG || $auditLog['action'] === self::ACTION_TAG_LOCAL); + $local = ($auditLog['action'] === self::ACTION_TAG_LOCAL || $auditLog['action'] === self::ACTION_REMOVE_TAG_LOCAL) ? __('local') : __('global'); + if ($attached) { + return __('Attached %s tag "%s" to %s #%s', $local, $auditLog['model_title'], strtolower($auditLog['model']), $auditLog['model_id']); + } else { + return __('Detached %s tag "%s" from %s #%s', $local, $auditLog['model_title'], strtolower($auditLog['model']), $auditLog['model_id']); + } + } + + + $title = "{$auditLog['model']} #{$auditLog['model_id']}"; + + if (isset($auditLog['model_title']) && $auditLog['model_title']) { + $title .= ": {$auditLog['model_title']}"; + } + return $title; + } +} diff --git a/src/Model/Table/AuditLogsTable.php b/src/Model/Table/AuditLogsTable.php new file mode 100644 index 0000000..374a156 --- /dev/null +++ b/src/Model/Table/AuditLogsTable.php @@ -0,0 +1,250 @@ +addBehavior('UUID'); + $this->addBehavior('Timestamp', [ + 'Model.beoreSave' => [ + 'created_at' => 'new' + ] + ]); + $this->belongsTo('Users'); + $this->setDisplayField('info'); + $this->compressionEnabled = Configure::read('Cerebrate.log_new_audit_compress') && function_exists('brotli_compress'); + } + + public function beforeMarshal(EventInterface $event, ArrayObject $data, ArrayObject $options) + { + if (!isset($data['request_ip'])) { + $ipHeader = 'REMOTE_ADDR'; + if (isset($_SERVER[$ipHeader])) { + $data['request_ip'] = $_SERVER[$ipHeader]; + } else { + $data['request_ip'] = '127.0.0.1'; + } + } + foreach (['user_id', 'request_type', 'authkey_id'] as $field) { + if (!isset($data[$field])) { + if (!isset($userInfo)) { + $userInfo = $this->userInfo(); + } + if (!empty($userInfo[$field])) { + $data[$field] = $userInfo[$field]; + } else { + $data[$field] = 0; + } + } + } + + if (!isset($data['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) { + $data['request_id'] = $_SERVER['HTTP_X_REQUEST_ID']; + } + + // Truncate request_id + if (isset($data['request_id']) && strlen($data['request_id']) > 255) { + $data['request_id'] = substr($data['request_id'], 0, 255); + } + + // Truncate model title + if (isset($data['model_title']) && mb_strlen($data['model_title']) > 255) { + $data['model_title'] = mb_substr($data['model_title'], 0, 252) . '...'; + } + + if (isset($data['change'])) { + $change = json_encode($data['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) { + $change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT); + } + $data['change'] = $change; + } + } + + public function beforeSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $entity->request_ip = inet_pton($entity->request_ip); + $this->logData($entity); + return true; + } + + /** + * @param array $data + * @return bool + */ + private function logData(EntityInterface $entity) + { + if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) { + $pubSubTool = $this->getPubSubTool(); + $pubSubTool->publish($data, 'audit', 'log'); + } + + //$this->publishKafkaNotification('audit', $data, 'log'); + + if (Configure::read('Plugin.ElasticSearch_logging_enable')) { + // send off our logs to distributed /dev/null + $logIndex = Configure::read("Plugin.ElasticSearch_log_index"); + $elasticSearchClient = $this->getElasticSearchTool(); + $elasticSearchClient->pushDocument($logIndex, "log", $data); + } + + // write to syslogd as well if enabled + if ($this->syslog === null) { + if (Configure::read('Security.syslog')) { + $options = []; + $syslogToStdErr = Configure::read('Security.syslog_to_stderr'); + if ($syslogToStdErr !== null) { + $options['to_stderr'] = $syslogToStdErr; + } + $syslogIdent = Configure::read('Security.syslog_ident'); + if ($syslogIdent) { + $options['ident'] = $syslogIdent; + } + $this->syslog = new SysLog($options); + } else { + $this->syslog = false; + } + } + if ($this->syslog) { + $entry = $data['action']; + $title = $entity->generateUserFriendlyTitle(); + if ($title) { + $entry .= " -- $title"; + } + $this->syslog->write('info', $entry); + } + return true; + } + + /** + * @return array + */ + public function userInfo() + { + if ($this->user !== null) { + return $this->user; + } + + $this->user = ['id' => 0, /*'org_id' => 0, */'authkey_id' => 0, 'request_type' => self::REQUEST_TYPE_DEFAULT]; + + $isShell = (php_sapi_name() === 'cli'); + if ($isShell) { + // do not start session for shell commands and fetch user info from configuration + $this->user['request_type'] = self::REQUEST_TYPE_CLI; + $currentUserId = Configure::read('CurrentUserId'); + if (!empty($currentUserId)) { + $this->user['id'] = $currentUserId; + $userFromDb = $this->Users->find()->where(['id' => $currentUserId])->first(); + $this->user['name'] = $userFromDb['name']; + $this->user['org_id'] = $userFromDb['org_id']; + } + } else { + $authUser = Router::getRequest()->getSession()->read('authUser'); + if (!empty($authUser)) { + $this->user['id'] = $authUser['id']; + $this->user['user_id'] = $authUser['id']; + $this->user['name'] = $authUser['name']; + //$this->user['org_id'] = $authUser['org_id']; + if (isset($authUser['logged_by_authkey']) && $authUser['logged_by_authkey']) { + $this->user['request_type'] = self::REQUEST_TYPE_API; + } + if (isset($authUser['authkey_id'])) { + $this->user['authkey_id'] = $authUser['authkey_id']; + } + } + } + return $this->user; + } + + public function insert(array $data) + { + $logEntity = $this->newEntity($data); + if ($logEntity->getErrors()) { + throw new Exception($logEntity->getErrors()); + } else { + $this->save($logEntity); + } + } + + /** + * @param string|int $org + * @return array + */ + public function returnDates($org = 'all') + { + $conditions = []; + if ($org !== 'all') { + $org = $this->Organisation->fetchOrg($org); + if (empty($org)) { + throw new NotFoundException('Invalid organisation.'); + } + $conditions['org_id'] = $org['id']; + } + + $dataSource = ConnectionManager::getDataSource('default')->config['datasource']; + if ($dataSource === 'Database/Mysql' || $dataSource === 'Database/MysqlObserver') { + $validDates = $this->find('all', [ + 'recursive' => -1, + 'fields' => ['DISTINCT UNIX_TIMESTAMP(DATE(created)) AS Date', 'count(id) AS count'], + 'conditions' => $conditions, + 'group' => ['Date'], + 'order' => ['Date'], + ]); + } elseif ($dataSource === 'Database/Postgres') { + if (!empty($conditions['org_id'])) { + $condOrg = sprintf('WHERE org_id = %s', intval($conditions['org_id'])); + } else { + $condOrg = ''; + } + $sql = 'SELECT DISTINCT EXTRACT(EPOCH FROM CAST(created AS DATE)) AS "Date", COUNT(id) AS count + FROM audit_logs + ' . $condOrg . ' + GROUP BY "Date" ORDER BY "Date"'; + $validDates = $this->query($sql); + } + $data = []; + foreach ($validDates as $date) { + $data[(int)$date[0]['Date']] = (int)$date[0]['count']; + } + return $data; + } +} From de2ee49ccf1cef213b9d96616b507d4160ab8c9b Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 14:44:54 +0100 Subject: [PATCH 16/27] new: [auditlogs] UI --- templates/AuditLogs/index.php | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 templates/AuditLogs/index.php diff --git a/templates/AuditLogs/index.php b/templates/AuditLogs/index.php new file mode 100644 index 0000000..46ec90b --- /dev/null +++ b/templates/AuditLogs/index.php @@ -0,0 +1,65 @@ +element('genericElements/IndexTable/index_table', [ + 'data' => [ + 'data' => $data, + 'top_bar' => [ + 'children' => [ + [ + 'type' => 'search', + 'button' => __('Filter'), + 'placeholder' => __('Enter value to search'), + 'data' => '', + 'searchKey' => 'value' + ] + ] + ], + 'fields' => [ + [ + 'name' => '#', + 'sort' => 'id', + 'data_path' => 'id', + ], + [ + 'name' => __('IP'), + 'sort' => 'request_ip', + 'data_path' => 'request_ip', + ], + [ + 'name' => __('Username'), + 'sort' => 'user.username', + 'data_path' => 'user.username', + ], + [ + 'name' => __('Title'), + 'data_path' => 'title', + ], + [ + 'name' => __('Model'), + 'sort' => 'model', + 'data_path' => 'model', + ], + [ + 'name' => __('Model ID'), + 'sort' => 'model', + 'data_path' => 'model_id', + ], + [ + 'name' => __('Action'), + 'sort' => 'action', + 'data_path' => 'action', + ], + [ + 'name' => __('Change'), + 'sort' => 'change', + 'data_path' => 'change', + 'element' => 'json' + ], + ], + 'title' => __('Logs'), + 'description' => null, + 'pull' => 'right', + 'actions' => [] + ] +]); +echo ''; +?> From af4f114f2f1928d933e556ecc016802d0d52f928 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 14:45:20 +0100 Subject: [PATCH 17/27] chg: [audit logs] tied into side menu --- src/Controller/Component/Navigation/sidemenu.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Controller/Component/Navigation/sidemenu.php b/src/Controller/Component/Navigation/sidemenu.php index 16dd350..fcca213 100644 --- a/src/Controller/Component/Navigation/sidemenu.php +++ b/src/Controller/Component/Navigation/sidemenu.php @@ -113,6 +113,11 @@ class Sidemenu { 'url' => '/instance/migrationIndex', 'icon' => 'database', ], + 'AuditLogs' => [ + 'label' => __('Audit Logs'), + 'url' => '/auditLogs/index', + 'icon' => 'history', + ], ] ], ], @@ -144,4 +149,4 @@ class Sidemenu { ] ]; } -} \ No newline at end of file +} From a305bdf9f192e448654091b3c5e7e0e82c958fbe Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 14:45:45 +0100 Subject: [PATCH 18/27] new: [mysql] added new table for audit logs --- INSTALL/mysql.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/INSTALL/mysql.sql b/INSTALL/mysql.sql index 88a3040..e5dcd7a 100644 --- a/INSTALL/mysql.sql +++ b/INSTALL/mysql.sql @@ -390,6 +390,26 @@ CREATE TABLE `meta_template_fields` ( KEY `type` (`type`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +CREATE TABLE IF NOT EXISTS `audit_logs` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `created` datetime NOT NULL, + `user_id` int(10) unsigned NOT NULL, + `authkey_id` int(11) DEFAULT NULL, + `ip` varbinary(16) DEFAULT NULL, + `request_type` tinyint NOT NULL, + `request_id` varchar(191) DEFAULT NULL, + `action` varchar(20) NOT NULL, + `model` varchar(80) NOT NULL, + `model_id` int(10) unsigned NOT NULL, + `model_title` text DEFAULT NULL, + `change` blob, + PRIMARY KEY (`id`), + KEY `user_id` (`user_id`), + KEY `ip` (`ip`), + KEY `model` (`model`), + KEY `action` (`action`), + KEY `model_id` (`model_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; From 72bd5641201940cd5e6e0750fbb4a584f5b11639 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:40:44 +0100 Subject: [PATCH 19/27] new: [migration] scripts added - also updated mysql.sql --- INSTALL/mysql.sql | 11 ++- .../Migrations/20211117135403_audit_logs.php | 96 +++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 config/Migrations/20211117135403_audit_logs.php diff --git a/INSTALL/mysql.sql b/INSTALL/mysql.sql index e5dcd7a..6bd956c 100644 --- a/INSTALL/mysql.sql +++ b/INSTALL/mysql.sql @@ -393,14 +393,14 @@ CREATE TABLE `meta_template_fields` ( CREATE TABLE IF NOT EXISTS `audit_logs` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `created` datetime NOT NULL, - `user_id` int(10) unsigned NOT NULL, - `authkey_id` int(11) DEFAULT NULL, - `ip` varbinary(16) DEFAULT NULL, + `user_id` int(10) unsigned DEFAULT NULL, + `authkey_id` int(10) unsigned DEFAULT NULL, + `request_ip` varbinary(16) DEFAULT NULL, `request_type` tinyint NOT NULL, `request_id` varchar(191) DEFAULT NULL, - `action` varchar(20) NOT NULL, + `request_action` varchar(20) NOT NULL, `model` varchar(80) NOT NULL, - `model_id` int(10) unsigned NOT NULL, + `model_id` int(10) unsigned DEFAULT NULL, `model_title` text DEFAULT NULL, `change` blob, PRIMARY KEY (`id`), @@ -409,6 +409,7 @@ CREATE TABLE IF NOT EXISTS `audit_logs` ( KEY `model` (`model`), KEY `action` (`action`), KEY `model_id` (`model_id`) + KEY `created` (`created`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; diff --git a/config/Migrations/20211117135403_audit_logs.php b/config/Migrations/20211117135403_audit_logs.php new file mode 100644 index 0000000..c44847b --- /dev/null +++ b/config/Migrations/20211117135403_audit_logs.php @@ -0,0 +1,96 @@ +hasTable('audit_logs'); + if (!$exists) { + $table = $this->table('audit_logs', [ + 'signed' => false, + 'collation' => 'utf8mb4_unicode_ci' + ]); + $table + ->addColumn('id', 'integer', [ + 'autoIncrement' => true, + 'limit' => 10, + 'signed' => false, + ]) + ->addPrimaryKey('id') + ->addColumn('user_id', 'integer', [ + 'default' => null, + 'null' => true, + 'signed' => false, + 'length' => 10 + ]) + ->addColumn('authkey_id', 'integer', [ + 'default' => null, + 'null' => true, + 'signed' => false, + 'length' => 10 + ]) + ->addColumn('request_ip', 'varbinary', [ + 'default' => null, + 'null' => true, + 'length' => 16 + ]) + ->addColumn('request_type', 'boolean', [ + 'null' => false + ]) + ->addColumn('request_id', 'integer', [ + 'default' => null, + 'null' => true, + 'signed' => false, + 'length' => 10 + ]) + ->addColumn('request_action', 'string', [ + 'null' => false, + 'length' => 20 + ]) + ->addColumn('model', 'string', [ + 'null' => false, + 'length' => 80 + ]) + ->addColumn('model_id', 'integer', [ + 'default' => null, + 'null' => true, + 'signed' => false, + 'length' => 10 + ]) + ->addColumn('model_title', 'text', [ + 'default' => null, + 'null' => true + ]) + ->addColumn('change', 'blob', [ + ]) + ->addColumn('created', 'datetime', [ + 'default' => null, + 'null' => false, + ]) + ->addIndex('user_id') + ->addIndex('request_ip') + ->addIndex('model') + ->addIndex('model_id') + ->addIndex('request_action') + ->addIndex('created'); + $table->create(); + } + } +} From 2e1ee2d064c07322d7f074160a6de9d5512e43f6 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:43:52 +0100 Subject: [PATCH 20/27] new: [audit log] behaviour tied into the appropriate models --- src/Model/Table/AlignmentsTable.php | 1 + src/Model/Table/AuditLogsTable.php | 2 +- src/Model/Table/AuthKeysTable.php | 3 ++- src/Model/Table/BroodsTable.php | 5 +++-- src/Model/Table/EncryptionKeysTable.php | 1 + src/Model/Table/InboxTable.php | 4 ++-- src/Model/Table/IndividualsTable.php | 1 + src/Model/Table/InstanceTable.php | 4 +++- src/Model/Table/LocalToolsTable.php | 1 + src/Model/Table/MetaFieldsTable.php | 1 + src/Model/Table/OrganisationsTable.php | 1 + src/Model/Table/OutboxProcessorsTable.php | 5 +++-- src/Model/Table/OutboxTable.php | 2 +- src/Model/Table/RemoteToolConnectionsTable.php | 1 + src/Model/Table/RolesTable.php | 1 + src/Model/Table/SettingsTable.php | 1 + src/Model/Table/SharingGroupsTable.php | 1 + src/Model/Table/UsersTable.php | 1 + 18 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/Model/Table/AlignmentsTable.php b/src/Model/Table/AlignmentsTable.php index 0914e99..c05ad90 100644 --- a/src/Model/Table/AlignmentsTable.php +++ b/src/Model/Table/AlignmentsTable.php @@ -12,6 +12,7 @@ class AlignmentsTable extends AppTable { parent::initialize($config); $this->belongsTo('Individuals'); + $this->addBehavior('AuditLog'); $this->belongsTo('Organisations'); $this->addBehavior('Timestamp'); } diff --git a/src/Model/Table/AuditLogsTable.php b/src/Model/Table/AuditLogsTable.php index 374a156..b65b5b2 100644 --- a/src/Model/Table/AuditLogsTable.php +++ b/src/Model/Table/AuditLogsTable.php @@ -145,7 +145,7 @@ class AuditLogsTable extends AppTable } } if ($this->syslog) { - $entry = $data['action']; + $entry = $data['request_action']; $title = $entity->generateUserFriendlyTitle(); if ($title) { $entry .= " -- $title"; diff --git a/src/Model/Table/AuthKeysTable.php b/src/Model/Table/AuthKeysTable.php index 3663af9..f5336e6 100644 --- a/src/Model/Table/AuthKeysTable.php +++ b/src/Model/Table/AuthKeysTable.php @@ -19,10 +19,11 @@ class AuthKeysTable extends AppTable { parent::initialize($config); $this->addBehavior('UUID'); + $this->addBehavior('AuditLog'); $this->belongsTo( 'Users' ); - $this->setDisplayField('authkey'); + $this->setDisplayField('comment'); } public function beforeMarshal(EventInterface $event, ArrayObject $data, ArrayObject $options) diff --git a/src/Model/Table/BroodsTable.php b/src/Model/Table/BroodsTable.php index e1993b4..b0d3dca 100644 --- a/src/Model/Table/BroodsTable.php +++ b/src/Model/Table/BroodsTable.php @@ -19,6 +19,7 @@ class BroodsTable extends AppTable parent::initialize($config); $this->addBehavior('UUID'); $this->addBehavior('Timestamp'); + $this->addBehavior('AuditLog'); $this->BelongsTo( 'Organisations' ); @@ -278,7 +279,7 @@ class BroodsTable extends AppTable } return $jsonReply; } - + /** * handleSendingFailed - Handle the case if the request could not be sent or if the remote rejected the connection request * @@ -302,7 +303,7 @@ class BroodsTable extends AppTable ]; return $creationResult; } - + /** * handleMessageNotCreated - Handle the case if the request was sent but the remote brood did not save the message in the inbox * diff --git a/src/Model/Table/EncryptionKeysTable.php b/src/Model/Table/EncryptionKeysTable.php index 23b4867..2008e0d 100644 --- a/src/Model/Table/EncryptionKeysTable.php +++ b/src/Model/Table/EncryptionKeysTable.php @@ -14,6 +14,7 @@ class EncryptionKeysTable extends AppTable { parent::initialize($config); $this->addBehavior('UUID'); + $this->addBehavior('AuditLog'); $this->addBehavior('Timestamp'); $this->belongsTo( 'Individuals', diff --git a/src/Model/Table/InboxTable.php b/src/Model/Table/InboxTable.php index 0c62a80..18faf47 100644 --- a/src/Model/Table/InboxTable.php +++ b/src/Model/Table/InboxTable.php @@ -19,7 +19,7 @@ class InboxTable extends AppTable parent::initialize($config); $this->addBehavior('UUID'); $this->addBehavior('Timestamp'); - + $this->addBehavior('AuditLog'); $this->belongsTo('Users'); $this->setDisplayField('title'); } @@ -68,7 +68,7 @@ class InboxTable extends AppTable if (empty($brood)) { $errors[] = __('Unkown brood `{0}`', $entryData['data']['cerebrateURL']); } - + // $found = false; // foreach ($user->individual->organisations as $organisations) { // if ($organisations->id == $brood->organisation_id) { diff --git a/src/Model/Table/IndividualsTable.php b/src/Model/Table/IndividualsTable.php index f0ba07f..70f63fd 100644 --- a/src/Model/Table/IndividualsTable.php +++ b/src/Model/Table/IndividualsTable.php @@ -16,6 +16,7 @@ class IndividualsTable extends AppTable $this->addBehavior('UUID'); $this->addBehavior('Timestamp'); $this->addBehavior('Tags.Tag'); + $this->addBehavior('AuditLog'); $this->hasMany( 'Alignments', [ diff --git a/src/Model/Table/InstanceTable.php b/src/Model/Table/InstanceTable.php index 0ec2a53..a625e79 100644 --- a/src/Model/Table/InstanceTable.php +++ b/src/Model/Table/InstanceTable.php @@ -18,6 +18,8 @@ class InstanceTable extends AppTable public function initialize(array $config): void { parent::initialize($config); + $this->addBehavior('AuditLog'); + $this->setDisplayField('name'); } public function validationDefault(Validator $validator): Validator @@ -54,7 +56,7 @@ class InstanceTable extends AppTable $timeline[$entry->date]['count'] = $entry->count; } $statistics[$model]['timeline'] = array_values($timeline); - + $startCount = $table->find()->where(['modified <' => new \DateTime("-{$days} days")])->all()->count(); $endCount = $statistics[$model]['amount']; $statistics[$model]['variation'] = $endCount - $startCount; diff --git a/src/Model/Table/LocalToolsTable.php b/src/Model/Table/LocalToolsTable.php index 8f2bfea..ac29bf9 100644 --- a/src/Model/Table/LocalToolsTable.php +++ b/src/Model/Table/LocalToolsTable.php @@ -30,6 +30,7 @@ class LocalToolsTable extends AppTable public function initialize(array $config): void { parent::initialize($config); + $this->addBehavior('AuditLog'); $this->addBehavior('Timestamp'); } diff --git a/src/Model/Table/MetaFieldsTable.php b/src/Model/Table/MetaFieldsTable.php index ba4cc66..bdcb212 100644 --- a/src/Model/Table/MetaFieldsTable.php +++ b/src/Model/Table/MetaFieldsTable.php @@ -12,6 +12,7 @@ class MetaFieldsTable extends AppTable { parent::initialize($config); $this->addBehavior('UUID'); + $this->addBehavior('AuditLog'); $this->setDisplayField('field'); $this->belongsTo('MetaTemplates'); $this->belongsTo('MetaTemplateFields'); diff --git a/src/Model/Table/OrganisationsTable.php b/src/Model/Table/OrganisationsTable.php index 254a570..d4a99b9 100644 --- a/src/Model/Table/OrganisationsTable.php +++ b/src/Model/Table/OrganisationsTable.php @@ -20,6 +20,7 @@ class OrganisationsTable extends AppTable parent::initialize($config); $this->addBehavior('Timestamp'); $this->addBehavior('Tags.Tag'); + $this->addBehavior('AuditLog'); $this->hasMany( 'Alignments', [ diff --git a/src/Model/Table/OutboxProcessorsTable.php b/src/Model/Table/OutboxProcessorsTable.php index da26692..5812cb4 100644 --- a/src/Model/Table/OutboxProcessorsTable.php +++ b/src/Model/Table/OutboxProcessorsTable.php @@ -28,6 +28,7 @@ class OutboxProcessorsTable extends AppTable if (empty($this->outboxProcessors)) { $this->loadProcessors(); } + $this->addBehavior('AuditLog'); } public function getProcessor($scope, $action=null) @@ -87,7 +88,7 @@ class OutboxProcessorsTable extends AppTable } } } - + /** * getProcessorClass * @@ -112,7 +113,7 @@ class OutboxProcessorsTable extends AppTable return $e->getMessage(); } } - + /** * createOutboxEntry * diff --git a/src/Model/Table/OutboxTable.php b/src/Model/Table/OutboxTable.php index 02fd442..a78e0c6 100644 --- a/src/Model/Table/OutboxTable.php +++ b/src/Model/Table/OutboxTable.php @@ -19,8 +19,8 @@ class OutboxTable extends AppTable parent::initialize($config); $this->addBehavior('UUID'); $this->addBehavior('Timestamp'); - $this->belongsTo('Users'); + $this->addBehavior('AuditLog'); $this->setDisplayField('title'); } diff --git a/src/Model/Table/RemoteToolConnectionsTable.php b/src/Model/Table/RemoteToolConnectionsTable.php index 7e8cf21..1e1ee25 100644 --- a/src/Model/Table/RemoteToolConnectionsTable.php +++ b/src/Model/Table/RemoteToolConnectionsTable.php @@ -18,6 +18,7 @@ class RemoteToolConnectionsTable extends AppTable 'LocalTools' ); $this->setDisplayField('id'); + $this->addBehavior('AuditLog'); } public function validationDefault(Validator $validator): Validator diff --git a/src/Model/Table/RolesTable.php b/src/Model/Table/RolesTable.php index 3973b1b..74f290b 100644 --- a/src/Model/Table/RolesTable.php +++ b/src/Model/Table/RolesTable.php @@ -12,6 +12,7 @@ class RolesTable extends AppTable { parent::initialize($config); $this->addBehavior('UUID'); + $this->addBehavior('AuditLog'); $this->hasMany( 'Users', [ diff --git a/src/Model/Table/SettingsTable.php b/src/Model/Table/SettingsTable.php index c2d71c9..7e9bcff 100644 --- a/src/Model/Table/SettingsTable.php +++ b/src/Model/Table/SettingsTable.php @@ -26,6 +26,7 @@ class SettingsTable extends AppTable parent::initialize($config); $this->setTable(false); $this->SettingsProvider = new CerebrateSettingsProvider(); + $this->addBehavior('AuditLog'); } public function getSettings($full=false): array diff --git a/src/Model/Table/SharingGroupsTable.php b/src/Model/Table/SharingGroupsTable.php index ec3791e..ff39220 100644 --- a/src/Model/Table/SharingGroupsTable.php +++ b/src/Model/Table/SharingGroupsTable.php @@ -15,6 +15,7 @@ class SharingGroupsTable extends AppTable parent::initialize($config); $this->addBehavior('UUID'); $this->addBehavior('Timestamp'); + $this->addBehavior('AuditLog'); $this->belongsTo( 'Users' ); diff --git a/src/Model/Table/UsersTable.php b/src/Model/Table/UsersTable.php index d9804a4..3a98b00 100644 --- a/src/Model/Table/UsersTable.php +++ b/src/Model/Table/UsersTable.php @@ -20,6 +20,7 @@ class UsersTable extends AppTable parent::initialize($config); $this->addBehavior('Timestamp'); $this->addBehavior('UUID'); + $this->addBehavior('AuditLog'); $this->initAuthBehaviors(); $this->belongsTo( 'Individuals', From 1f7756934487a44e2363ef7747953da9d537db3e Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:46:32 +0100 Subject: [PATCH 21/27] chg: [auditlog] log api authentication failures / successes --- src/Controller/AppController.php | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Controller/AppController.php b/src/Controller/AppController.php index 3771796..9b44ec4 100644 --- a/src/Controller/AppController.php +++ b/src/Controller/AppController.php @@ -111,6 +111,7 @@ class AppController extends Controller } unset($user['password']); $this->ACL->setUser($user); + $this->request->getSession()->write('authUser', $user); $this->isAdmin = $user['role']['perm_admin']; $this->set('menu', $this->ACL->getMenu()); $this->set('loggedUser', $this->ACL->getUser()); @@ -147,13 +148,31 @@ class AppController extends Controller { if (!empty($_SERVER['HTTP_AUTHORIZATION']) && strlen($_SERVER['HTTP_AUTHORIZATION'])) { $this->loadModel('AuthKeys'); + $logModel = $this->Users->auditLogs(); $authKey = $this->AuthKeys->checkKey($_SERVER['HTTP_AUTHORIZATION']); if (!empty($authKey)) { $this->loadModel('Users'); $user = $this->Users->get($authKey['user_id']); + $user = $logModel->userInfo(); + $logModel->insert([ + 'action' => 'login', + 'model' => 'Users', + 'model_id' => $user['id'], + 'model_title' => $user['name'], + 'change' => [] + ]); if (!empty($user)) { $this->Authentication->setIdentity($user); } + } else { + $user = $logModel->userInfo(); + $logModel->insert([ + 'action' => 'login', + 'model' => 'Users', + 'model_id' => $user['id'], + 'model_title' => $user['name'], + 'change' => [] + ]); } } } From cc043733752a5e1e660a1df761267e4d008a8332 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:47:32 +0100 Subject: [PATCH 22/27] new: [crud component] fixes - add hidden option - fix afterfind --- src/Controller/Component/CRUDComponent.php | 29 +++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Controller/Component/CRUDComponent.php b/src/Controller/Component/CRUDComponent.php index 64b5777..45df6d5 100644 --- a/src/Controller/Component/CRUDComponent.php +++ b/src/Controller/Component/CRUDComponent.php @@ -61,11 +61,25 @@ class CRUDComponent extends Component } if ($this->Controller->ParamHandler->isRest()) { $data = $query->all(); + if (isset($options['hidden'])) { + $data->each(function($value, $key) use ($options) { + $hidden = is_array($options['hidden']) ? $options['hidden'] : [$options['hidden']]; + $value->setHidden($hidden); + return $value; + }); + } if (isset($options['afterFind'])) { + $function = $options['afterFind']; if (is_callable($options['afterFind'])) { - $data = $options['afterFind']($data); + $function = $options['afterFind']; + $data->each(function($value, $key) use ($function) { + return $function($value); + }); } else { - $data = $this->Table->{$options['afterFind']}($data); + $t = $this->Table; + $data->each(function($value, $key) use ($t, $function) { + return $t->$function($value); + }); } } $this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json'); @@ -73,10 +87,17 @@ class CRUDComponent extends Component $this->Controller->loadComponent('Paginator'); $data = $this->Controller->Paginator->paginate($query); if (isset($options['afterFind'])) { + $function = $options['afterFind']; if (is_callable($options['afterFind'])) { - $data = $options['afterFind']($data); + $function = $options['afterFind']; + $data->each(function($value, $key) use ($function) { + return $function($value); + }); } else { - $data = $this->Table->{$options['afterFind']}($data); + $t = $this->Table; + $data->each(function($value, $key) use ($t, $function) { + return $t->$function($value); + }); } } $this->setFilteringContext($options['contextFilters'] ?? [], $params); From bc2e2fa4887703f21ded8d8856c25b802adac304 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:48:49 +0100 Subject: [PATCH 23/27] new: [open] individualscontroller fix - import badrequest exception --- src/Controller/Open/IndividualsController.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Controller/Open/IndividualsController.php b/src/Controller/Open/IndividualsController.php index fa75aaa..28cd51d 100644 --- a/src/Controller/Open/IndividualsController.php +++ b/src/Controller/Open/IndividualsController.php @@ -6,6 +6,7 @@ use App\Controller\AppController; use Cake\Utility\Hash; use Cake\Utility\Text; use \Cake\Database\Expression\QueryExpression; +use Cake\Http\Exception\BadRequestException; use Cake\Http\Exception\NotFoundException; use Cake\Http\Exception\MethodNotAllowedException; use Cake\Http\Exception\ForbiddenException; From 7b52d29320365b3df6b9ae6ebea6bba5e27849f3 Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:49:28 +0100 Subject: [PATCH 24/27] new: [login] log success/failure --- src/Controller/UsersController.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/Controller/UsersController.php b/src/Controller/UsersController.php index e5258d0..012655e 100644 --- a/src/Controller/UsersController.php +++ b/src/Controller/UsersController.php @@ -130,11 +130,27 @@ class UsersController extends AppController { $result = $this->Authentication->getResult(); // If the user is logged in send them away. + $logModel = $this->Users->auditLogs(); if ($result->isValid()) { + $user = $logModel->userInfo(); + $logModel->insert([ + 'request_action' => 'login', + 'model' => 'Users', + 'model_id' => $user['id'], + 'model_title' => $user['name'], + 'change' => [] + ]); $target = $this->Authentication->getLoginRedirect() ?? '/instance/home'; return $this->redirect($target); } if ($this->request->is('post') && !$result->isValid()) { + $logModel->insert([ + 'request_action' => 'login_fail', + 'model' => 'Users', + 'model_id' => 0, + 'model_title' => 'unknown_user', + 'change' => [] + ]); $this->Flash->error(__('Invalid username or password')); } $this->viewBuilder()->setLayout('login'); @@ -144,6 +160,15 @@ class UsersController extends AppController { $result = $this->Authentication->getResult(); if ($result->isValid()) { + $logModel = $this->Users->auditLogs(); + $user = $logModel->userInfo(); + $logModel->insert([ + 'request_action' => 'logout', + 'model' => 'Users', + 'model_id' => $user['id'], + 'model_title' => $user['name'], + 'change' => [] + ]); $this->Authentication->logout(); $this->Flash->success(__('Goodbye.')); return $this->redirect(\Cake\Routing\Router::url('/users/login')); From fe8e217d61d4cd480bcc0568d684fbf87dfe4e8f Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:57:34 +0100 Subject: [PATCH 25/27] chg: [audit log naming] renamed action to request_action to avoid reserved keyword usage --- src/Model/Behavior/AuditLogBehavior.php | 4 ++-- src/Model/Entity/AuditLog.php | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Model/Behavior/AuditLogBehavior.php b/src/Model/Behavior/AuditLogBehavior.php index 3eacfe2..f7ef3df 100644 --- a/src/Model/Behavior/AuditLogBehavior.php +++ b/src/Model/Behavior/AuditLogBehavior.php @@ -106,7 +106,7 @@ class AuditLogBehavior extends Behavior $modelTitle = $this->old[$modelTitleField]; } $this->auditLogs()->insert([ - 'action' => $action, + 'request_action' => $action, 'model' => $entity->getSource(), 'model_id' => $id, 'model_title' => $modelTitle, @@ -130,7 +130,7 @@ class AuditLogBehavior extends Behavior } $logEntity = $this->auditLogs()->newEntity([ - 'action' => $entity->getConstant('ACTION_DELETE'), + 'request_action' => $entity->getConstant('ACTION_DELETE'), 'model' => $entity->getSource(), 'model_id' => $this->old->id, 'model_title' => $modelTitle, diff --git a/src/Model/Entity/AuditLog.php b/src/Model/Entity/AuditLog.php index beabeb8..74e2f41 100644 --- a/src/Model/Entity/AuditLog.php +++ b/src/Model/Entity/AuditLog.php @@ -47,9 +47,9 @@ class AuditLog extends AppModel */ public function generateUserFriendlyTitle($auditLog) { - if (in_array($auditLog['action'], [self::ACTION_TAG, self::ACTION_TAG_LOCAL, self::ACTION_REMOVE_TAG, self::ACTION_REMOVE_TAG_LOCAL], true)) { - $attached = ($auditLog['action'] === self::ACTION_TAG || $auditLog['action'] === self::ACTION_TAG_LOCAL); - $local = ($auditLog['action'] === self::ACTION_TAG_LOCAL || $auditLog['action'] === self::ACTION_REMOVE_TAG_LOCAL) ? __('local') : __('global'); + if (in_array($auditLog['request_action'], [self::ACTION_TAG, self::ACTION_TAG_LOCAL, self::ACTION_REMOVE_TAG, self::ACTION_REMOVE_TAG_LOCAL], true)) { + $attached = ($auditLog['request_action'] === self::ACTION_TAG || $auditLog['request_action'] === self::ACTION_TAG_LOCAL); + $local = ($auditLog['request_action'] === self::ACTION_TAG_LOCAL || $auditLog['request_action'] === self::ACTION_REMOVE_TAG_LOCAL) ? __('local') : __('global'); if ($attached) { return __('Attached %s tag "%s" to %s #%s', $local, $auditLog['model_title'], strtolower($auditLog['model']), $auditLog['model_id']); } else { From ff77af0a8edabf2d62bcc3237143a0c5b7555b1f Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:58:06 +0100 Subject: [PATCH 26/27] new: [appmodel] moved constants related to the logging along with a getter to app model --- src/Model/Entity/AppModel.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/Model/Entity/AppModel.php b/src/Model/Entity/AppModel.php index 84c86b3..50bfe86 100644 --- a/src/Model/Entity/AppModel.php +++ b/src/Model/Entity/AppModel.php @@ -6,6 +6,28 @@ use Cake\ORM\Entity; class AppModel extends Entity { + const BROTLI_HEADER = "\xce\xb2\xcf\x81"; + const BROTLI_MIN_LENGTH = 200; + + const ACTION_ADD = 'add', + ACTION_EDIT = 'edit', + ACTION_SOFT_DELETE = 'soft_delete', + ACTION_DELETE = 'delete', + ACTION_UNDELETE = 'undelete', + ACTION_TAG = 'tag', + ACTION_TAG_LOCAL = 'tag_local', + ACTION_REMOVE_TAG = 'remove_tag', + ACTION_REMOVE_TAG_LOCAL = 'remove_local_tag', + ACTION_LOGIN = 'login', + ACTION_LOGIN_FAIL = 'login_fail', + ACTION_LOGOUT = 'logout'; + + + public function getConstant($name) + { + return constant('self::' . $name); + } + public function getAccessibleFieldForNew(): array { return $this->_accessibleOnNew ?? []; From 92ddd04ba0c4bfde46c69f3ecb996cd5f80a4b8c Mon Sep 17 00:00:00 2001 From: iglocska Date: Wed, 17 Nov 2021 15:58:52 +0100 Subject: [PATCH 27/27] fix: [JSON fields] fixed escaping issues --- templates/element/genericElements/IndexTable/Fields/json.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/element/genericElements/IndexTable/Fields/json.php b/templates/element/genericElements/IndexTable/Fields/json.php index 6c14248..2aca821 100644 --- a/templates/element/genericElements/IndexTable/Fields/json.php +++ b/templates/element/genericElements/IndexTable/Fields/json.php @@ -1,9 +1,12 @@ Hash->extract($row, $field['data_path'])); + $data = $this->Hash->extract($row, $field['data_path']); // I feed dirty for this... if (is_array($data) && count($data) === 1 && isset($data[0])) { $data = $data[0]; } + if (!is_array($data)) { + $data = json_decode($data, true); + } echo sprintf( '
', h($k)