diff --git a/.gitchangelog.rc b/.gitchangelog.rc new file mode 100644 index 0000000..053a539 --- /dev/null +++ b/.gitchangelog.rc @@ -0,0 +1,289 @@ +# -*- coding: utf-8; mode: python -*- +## +## Format +## +## ACTION: [AUDIENCE:] COMMIT_MSG [!TAG ...] +## +## Description +## +## ACTION is one of 'chg', 'fix', 'new' +## +## Is WHAT the change is about. +## +## 'chg' is for refactor, small improvement, cosmetic changes... +## 'fix' is for bug fixes +## 'new' is for new features, big improvement +## +## AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'|'docs' +## +## Is WHO is concerned by the change. +## +## 'dev' is for developpers (API changes, refactors...) +## 'usr' is for final users (UI changes) +## 'pkg' is for packagers (packaging changes) +## 'test' is for testers (test only related changes) +## 'doc' is for doc guys (doc only changes) +## +## COMMIT_MSG is ... well ... the commit message itself. +## +## TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic' +## +## They are preceded with a '!' or a '@' (prefer the former, as the +## latter is wrongly interpreted in github.) Commonly used tags are: +## +## 'refactor' is obviously for refactoring code only +## 'minor' is for a very meaningless change (a typo, adding a comment) +## 'cosmetic' is for cosmetic driven change (re-indentation, 80-col...) +## 'wip' is for partial functionality but complete subfunctionality. +## +## Example: +## +## new: usr: support of bazaar implemented +## chg: re-indentend some lines !cosmetic +## new: dev: updated code to be compatible with last version of killer lib. +## fix: pkg: updated year of licence coverage. +## new: test: added a bunch of test around user usability of feature X. +## fix: typo in spelling my name in comment. !minor +## +## Please note that multi-line commit message are supported, and only the +## first line will be considered as the "summary" of the commit message. So +## tags, and other rules only applies to the summary. The body of the commit +## message will be displayed in the changelog without reformatting. + + +## +## ``ignore_regexps`` is a line of regexps +## +## Any commit having its full commit message matching any regexp listed here +## will be ignored and won't be reported in the changelog. +## +ignore_regexps = [ + r'@minor', r'!minor', + r'@cosmetic', r'!cosmetic', + r'@refactor', r'!refactor', + r'@wip', r'!wip', + r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[p|P]kg:', + r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[d|D]ev:', + r'^(.{3,3}\s*:)?\s*[fF]irst commit.?\s*$', + ] + + +## ``section_regexps`` is a list of 2-tuples associating a string label and a +## list of regexp +## +## Commit messages will be classified in sections thanks to this. Section +## titles are the label, and a commit is classified under this section if any +## of the regexps associated is matching. +## +## Please note that ``section_regexps`` will only classify commits and won't +## make any changes to the contents. So you'll probably want to go check +## ``subject_process`` (or ``body_process``) to do some changes to the subject, +## whenever you are tweaking this variable. +## +section_regexps = [ + ('New', [ + r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$', + ]), + ('Changes', [ + r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$', + ]), + ('Fix', [ + r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$', + ]), + + ('Other', None ## Match all lines + ), + +] + + +## ``body_process`` is a callable +## +## This callable will be given the original body and result will +## be used in the changelog. +## +## Available constructs are: +## +## - any python callable that take one txt argument and return txt argument. +## +## - ReSub(pattern, replacement): will apply regexp substitution. +## +## - Indent(chars=" "): will indent the text with the prefix +## Please remember that template engines gets also to modify the text and +## will usually indent themselves the text if needed. +## +## - Wrap(regexp=r"\n\n"): re-wrap text in separate paragraph to fill 80-Columns +## +## - noop: do nothing +## +## - ucfirst: ensure the first letter is uppercase. +## (usually used in the ``subject_process`` pipeline) +## +## - final_dot: ensure text finishes with a dot +## (usually used in the ``subject_process`` pipeline) +## +## - strip: remove any spaces before or after the content of the string +## +## - SetIfEmpty(msg="No commit message."): will set the text to +## whatever given ``msg`` if the current text is empty. +## +## Additionally, you can `pipe` the provided filters, for instance: +#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') | Indent(chars=" ") +#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') +#body_process = noop +body_process = ReSub(r'((^|\n)[A-Z]\w+(-\w+)*: .*(\n\s+.*)*)+$', r'') | strip + + +## ``subject_process`` is a callable +## +## This callable will be given the original subject and result will +## be used in the changelog. +## +## Available constructs are those listed in ``body_process`` doc. +subject_process = (strip | + ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') | + SetIfEmpty("No commit message.") | ucfirst | final_dot) + + +## ``tag_filter_regexp`` is a regexp +## +## Tags that will be used for the changelog must match this regexp. +## +tag_filter_regexp = r'^v[0-9]+\.[0-9]+$' + + + +## ``unreleased_version_label`` is a string or a callable that outputs a string +## +## This label will be used as the changelog Title of the last set of changes +## between last valid tag and HEAD if any. +unreleased_version_label = "%%version%% (unreleased)" + + +## ``output_engine`` is a callable +## +## This will change the output format of the generated changelog file +## +## Available choices are: +## +## - rest_py +## +## Legacy pure python engine, outputs ReSTructured text. +## This is the default. +## +## - mustache() +## +## Template name could be any of the available templates in +## ``templates/mustache/*.tpl``. +## Requires python package ``pystache``. +## Examples: +## - mustache("markdown") +## - mustache("restructuredtext") +## +## - makotemplate() +## +## Template name could be any of the available templates in +## ``templates/mako/*.tpl``. +## Requires python package ``mako``. +## Examples: +## - makotemplate("restructuredtext") +## +output_engine = rest_py +#output_engine = mustache("restructuredtext") +#output_engine = mustache("markdown") +#output_engine = makotemplate("restructuredtext") + + +## ``include_merge`` is a boolean +## +## This option tells git-log whether to include merge commits in the log. +## The default is to include them. +include_merge = True + + +## ``log_encoding`` is a string identifier +## +## This option tells gitchangelog what encoding is outputed by ``git log``. +## The default is to be clever about it: it checks ``git config`` for +## ``i18n.logOutputEncoding``, and if not found will default to git's own +## default: ``utf-8``. +#log_encoding = 'utf-8' + + +## ``publish`` is a callable +## +## Sets what ``gitchangelog`` should do with the output generated by +## the output engine. ``publish`` is a callable taking one argument +## that is an interator on lines from the output engine. +## +## Some helper callable are provided: +## +## Available choices are: +## +## - stdout +## +## Outputs directly to standard output +## (This is the default) +## +## - FileInsertAtFirstRegexMatch(file, pattern, idx=lamda m: m.start()) +## +## Creates a callable that will parse given file for the given +## regex pattern and will insert the output in the file. +## ``idx`` is a callable that receive the matching object and +## must return a integer index point where to insert the +## the output in the file. Default is to return the position of +## the start of the matched string. +## +## - FileRegexSubst(file, pattern, replace, flags) +## +## Apply a replace inplace in the given file. Your regex pattern must +## take care of everything and might be more complex. Check the README +## for a complete copy-pastable example. +## +# publish = FileInsertIntoFirstRegexMatch( +# "CHANGELOG.rst", +# r'/(?P[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n/', +# idx=lambda m: m.start(1) +# ) +#publish = stdout + + +## ``revs`` is a list of callable or a list of string +## +## callable will be called to resolve as strings and allow dynamical +## computation of these. The result will be used as revisions for +## gitchangelog (as if directly stated on the command line). This allows +## to filter exaclty which commits will be read by gitchangelog. +## +## To get a full documentation on the format of these strings, please +## refer to the ``git rev-list`` arguments. There are many examples. +## +## Using callables is especially useful, for instance, if you +## are using gitchangelog to generate incrementally your changelog. +## +## Some helpers are provided, you can use them:: +## +## - FileFirstRegexMatch(file, pattern): will return a callable that will +## return the first string match for the given pattern in the given file. +## If you use named sub-patterns in your regex pattern, it'll output only +## the string matching the regex pattern named "rev". +## +## - Caret(rev): will return the rev prefixed by a "^", which is a +## way to remove the given revision and all its ancestor. +## +## Please note that if you provide a rev-list on the command line, it'll +## replace this value (which will then be ignored). +## +## If empty, then ``gitchangelog`` will act as it had to generate a full +## changelog. +## +## The default is to use all commits to make the changelog. +#revs = ["^1.0.3", ] +#revs = [ +# Caret( +# FileFirstRegexMatch( +# "CHANGELOG.rst", +# r"(?P[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n")), +# "HEAD" +#] +revs = [] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7841c33 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# Temp files +*.swp +*.pyc +*.swo +*.o + +# redis datas +server/dump6380.rdb diff --git a/README.md b/README.md index 37177d5..5aa1d90 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,84 @@ # D4 core -Software components used for the D4 project +![](https://www.d4-project.org/assets/images/logo.png) + +D4 core are software components used in the D4 project. The software includes everything to create your own sensor network or connect +to an existing sensor network using simple clients. + +![https://github.com/D4-project/d4-core/releases/latest](https://img.shields.io/github/release/D4-project/d4-core/all.svg) +![https://github.com/D4-project/d4-core/blob/master/LICENSE](https://img.shields.io/badge/License-AGPL-yellow.svg) ## D4 core client +[D4 core client](https://github.com/D4-project/d4-core/tree/master/client) is a simple and minimal implementation of the [D4 encapsulation protocol](https://github.com/D4-project/architecture/tree/master/format). There is also a [portable D4 client](https://github.com/D4-project/d4-goclient) in Go including the support for the SSL/TLS connectivity. + +### Requirements + +- Unix-like operating system +- make +- a recent C compiler + +### Usage + +The D4 client can be used to stream any byte stream towards a D4 server. + +As an example, you directly stream tcpdump output to a D4 server with the following +script: + +```` +tcpdump -n -s0 -w - | ./d4 -c ./conf | socat - OPENSSL-CONNECT:$D4-SERVER-IP-ADDRESS:$PORT,verify=0 +```` + ~~~~ +d4 - d4 client +Read data from the configured and send it to + +Usage: d4 -c config_directory + +Configuration + +The configuration settings are stored in files in the configuration directory +specified with the -c command line switch. + +Files in the configuration directory + +key - is the private HMAC-SHA-256-128 key. + The HMAC is computed on the header with a HMAC value set to 0 + which is updated later. +snaplen - the length of bytes that is read from the +version - the version of the d4 client +type - the type of data that is send. pcap, netflow, ... +source - the source where the data is read from +destination - the destination where the data is written to +~~~~ + +### Installation + +~~~~ +cd client git submodule init git submodule update ~~~~ +## D4 core server + +D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of +sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers. + ### Requirements -- uuid-dev -- make -- a recent C compiler +- Python 3.6 +- GNU/Linux distribution +### Installation + + +- [Install D4 Server](https://github.com/D4-project/d4-core/tree/master/server) + +### Screenshots of D4 core server management + +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/main.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor-mgmt.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/analyzer-mgmt.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt2.png) diff --git a/doc/images/analyzer-mgmt.png b/doc/images/analyzer-mgmt.png new file mode 100644 index 0000000..7919696 Binary files /dev/null and b/doc/images/analyzer-mgmt.png differ diff --git a/doc/images/main.png b/doc/images/main.png new file mode 100644 index 0000000..c23e0cc Binary files /dev/null and b/doc/images/main.png differ diff --git a/doc/images/sensor-mgmt.png b/doc/images/sensor-mgmt.png new file mode 100644 index 0000000..0b20fff Binary files /dev/null and b/doc/images/sensor-mgmt.png differ diff --git a/doc/images/server-mgmt.png b/doc/images/server-mgmt.png new file mode 100644 index 0000000..01ced33 Binary files /dev/null and b/doc/images/server-mgmt.png differ diff --git a/doc/images/server-mgmt2.png b/doc/images/server-mgmt2.png new file mode 100644 index 0000000..e341743 Binary files /dev/null and b/doc/images/server-mgmt2.png differ diff --git a/server/.gitignore b/server/.gitignore index 7d17db7..aef579e 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -2,6 +2,7 @@ *.csr *.pem *.key +configs/server.conf data/ logs/ redis/ diff --git a/server/LAUNCH.sh b/server/LAUNCH.sh index 5a10d32..80c7044 100755 --- a/server/LAUNCH.sh +++ b/server/LAUNCH.sh @@ -36,7 +36,7 @@ function helptext { - D4 Twisted server. - All wokers manager. - All Redis in memory servers. - - Flak server. + - Flask server. Usage: LAUNCH.sh [-l | --launchAuto] @@ -51,7 +51,7 @@ function launching_redis { screen -dmS "Redis_D4" sleep 0.1 - echo -e $GREEN"\t* Launching D4 Redis ervers"$DEFAULT + echo -e $GREEN"\t* Launching D4 Redis Servers"$DEFAULT screen -S "Redis_D4" -X screen -t "6379" bash -c $redis_dir'redis-server '$conf_dir'6379.conf ; read x' sleep 0.1 screen -S "Redis_D4" -X screen -t "6380" bash -c $redis_dir'redis-server '$conf_dir'6380.conf ; read x' @@ -72,9 +72,13 @@ function launching_workers { sleep 0.1 echo -e $GREEN"\t* Launching D4 Workers"$DEFAULT - screen -S "Workers_D4" -X screen -t "1_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_1; ./workers_manager.py; read x" + screen -S "Workers_D4" -X screen -t "1_workers" bash -c "cd ${D4_HOME}/workers/workers_1; ./workers_manager.py; read x" sleep 0.1 - screen -S "Workers_D4" -X screen -t "4_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_4; ./workers_manager.py; read x" + screen -S "Workers_D4" -X screen -t "2_workers" bash -c "cd ${D4_HOME}/workers/workers_2; ./workers_manager.py; read x" + sleep 0.1 + screen -S "Workers_D4" -X screen -t "4_workers" bash -c "cd ${D4_HOME}/workers/workers_4; ./workers_manager.py; read x" + sleep 0.1 + screen -S "Workers_D4" -X screen -t "8_workers" bash -c "cd ${D4_HOME}/workers/workers_8; ./workers_manager.py; read x" sleep 0.1 } @@ -159,6 +163,7 @@ function launch_flask { screen -dmS "Flask_D4" sleep 0.1 echo -e $GREEN"\t* Launching Flask server"$DEFAULT + # screen -S "Flask_D4" -X screen -t "Flask_server" bash -c "cd $flask_dir; export FLASK_DEBUG=1;export FLASK_APP=Flask_server.py; python -m flask run --port 7000; read x" screen -S "Flask_D4" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" else echo -e $RED"\t* A Flask_D4 screen is already launched"$DEFAULT @@ -200,9 +205,15 @@ function update_web { fi } +function update_config { + echo -e $GREEN"\t* Updating Config File"$DEFAULT + bash -c "(cd ${D4_HOME}/configs; ./update_conf.py -v 0)" +} + function launch_all { helptext; launch_redis; + update_config; launch_d4_server; launch_workers; launch_flask; diff --git a/server/README.md b/server/README.md new file mode 100644 index 0000000..3035e1d --- /dev/null +++ b/server/README.md @@ -0,0 +1,68 @@ +# D4 core + +![](https://www.d4-project.org/assets/images/logo.png) + +## D4 core server + +D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of +sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers. + +### Requirements + +- Python 3.6 +- GNU/Linux distribution + +### Installation + +###### Install D4 server +~~~~ +cd server +./install_server.sh +~~~~ +Create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) : +~~~~ +cd gen_cert +./gen_root.sh +./gen_cert.sh +cd .. +~~~~ + + +###### Launch D4 server +~~~~ +./LAUNCH.sh -l +~~~~ + +The web interface is accessible via `http://127.0.0.1:7000/` + +### Updating web assets +To update javascript libs run: +~~~~ +cd web +./update_web.sh +~~~~ + +### Notes + +- All server logs are located in ``d4-core/server/logs/`` +- Close D4 Server: ``./LAUNCH.sh -k`` + +### Screenshots of D4 core server management + +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/main.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor-mgmt.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/analyzer-mgmt.png) +![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt2.png) + +### Troubleshooting + +###### Worker 1, tcpdump: Permission denied +Could be related to AppArmor: +~~~~ +sudo cat /var/log/syslog | grep denied +~~~~ +Run the following command as root: +~~~~ +aa-complain /usr/sbin/tcpdump +~~~~ diff --git a/server/configs/6379.conf b/server/configs/6379.conf index 8c7c469..25dd663 100644 --- a/server/configs/6379.conf +++ b/server/configs/6379.conf @@ -1,4 +1,9 @@ -# Redis configuration file example +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: @@ -15,7 +20,7 @@ ################################## INCLUDES ################################### # Include one or more other config files here. This is useful if you -# have a standard template that goes to all Redis server but also need +# have a standard template that goes to all Redis servers but also need # to customize a few per-server settings. Include files can include # other files, so use this wisely. # @@ -30,17 +35,59 @@ # include /path/to/local.conf # include /path/to/other.conf -################################ GENERAL ##################################### +################################## MODULES ##################################### -# By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize no +# Load modules at startup. If the server is not able to load modules +# it will abort. It is possible to use multiple loadmodule directives. +# +# loadmodule /path/to/my_module.so +# loadmodule /path/to/other_module.so -# When running daemonized, Redis writes a pid file in /var/run/redis.pid by -# default. You can specify a custom pid file location here. -#pidfile /var/run/redis.pid +################################## NETWORK ##################################### -# Accept connections on the specified port, default is 6379. +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all the network interfaces available on the server. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 ::1 +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only into +# the IPv4 loopback interface address (this means Redis will be able to +# accept connections only from clients running into the same computer it +# is running). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +bind 127.0.0.1 + +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode yes + +# Accept connections on the specified port, default is 6379 (IANA #815344). # If port 0 is specified Redis will not listen on a TCP socket. port 6379 @@ -53,22 +100,14 @@ port 6379 # in order to get the desired effect. tcp-backlog 511 -# By default Redis listens for connections from all the network interfaces -# available on the server. It is possible to listen to just one or multiple -# interfaces using the "bind" configuration directive, followed by one or -# more IP addresses. +# Unix socket. # -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 - # Specify the path for the Unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # -#unixsocket /tmp/redis.sock -#unixsocketperm 755 +# unixsocket /tmp/redis.sock +# unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) timeout 0 @@ -86,8 +125,37 @@ timeout 0 # Note that to close the connection the double of the time is needed. # On other kernels the period depends on the kernel configuration. # -# A reasonable value for this option is 60 seconds. -tcp-keepalive 0 +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize no + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised no + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +pidfile /var/run/redis_6379.pid # Specify the server verbosity level. # This can be one of: @@ -117,6 +185,14 @@ logfile "" # dbid is a number between 0 and 'databases'-1 databases 16 +# By default Redis shows an ASCII art logo only when started to log to the +# standard output and if the standard output is a TTY. Basically this means +# that normally a logo is displayed only in interactive sessions. +# +# However it is possible to force the pre-4.0 behavior and always show a +# ASCII art logo in startup logs by setting the following option to yes. +always-show-logo yes + ################################ SNAPSHOTTING ################################ # # Save the DB on disk: @@ -131,7 +207,7 @@ databases 16 # after 300 sec (5 min) if at least 10 keys changed # after 60 sec if at least 10000 keys changed # -# Note: you can disable saving at all commenting all the "save" lines. +# Note: you can disable saving completely by commenting out all "save" lines. # # It is also possible to remove all the previously configured save # points by adding a save directive with a single empty string argument @@ -188,140 +264,232 @@ dir ./ ################################# REPLICATION ################################# -# Master-Slave replication. Use slaveof to make a Redis instance a copy of -# another Redis server. Note that the configuration is local to the slave -# so for example it is possible to configure the slave to save the DB with a -# different interval, or to listen to another port, and so on. +# Master-Replica replication. Use replicaof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. # -# slaveof +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition replicas automatically try to reconnect to masters +# and resynchronize with them. +# +# replicaof # If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before +# directive below) it is possible to tell the replica to authenticate before # starting the replication synchronization process, otherwise the master will -# refuse the slave request. +# refuse the replica request. # # masterauth -# When a slave loses its connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: # -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# 2) if replica-serve-stale-data is set to 'no' the replica will reply with # an error "SYNC with master in progress" to all the kind of commands -# but to INFO and SLAVEOF. +# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, +# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, +# COMMAND, POST, HOST: and LATENCY. # -slave-serve-stale-data yes +replica-serve-stale-data yes -# You can configure a slave instance to accept writes or not. Writing against -# a slave instance may be useful to store some ephemeral data (because data -# written on a slave will be easily deleted after resync with the master) but +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but # may also cause problems if clients are writing to it because of a # misconfiguration. # -# Since Redis 2.6 by default slaves are read-only. +# Since Redis 2.6 by default replicas are read-only. # -# Note: read only slaves are not designed to be exposed to untrusted clients +# Note: read only replicas are not designed to be exposed to untrusted clients # on the internet. It's just a protection layer against misuse of the instance. -# Still a read only slave exports by default all the administrative commands +# Still a read only replica exports by default all the administrative commands # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve -# security of read only slaves using 'rename-command' to shadow all the +# security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. -slave-read-only yes +replica-read-only yes -# Slaves send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_slave_period option. The default value is 10 +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New replicas and reconnecting replicas that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the replicas. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the replicas incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to replica sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more replicas +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new replicas arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple replicas +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that transfers the RDB via socket +# to the replicas. +# +# This is important since once the transfer starts, it is not possible to serve +# new replicas arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more replicas arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Replicas send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_replica_period option. The default value is 10 # seconds. # -# repl-ping-slave-period 10 +# repl-ping-replica-period 10 # The following option sets the replication timeout for: # -# 1) Bulk transfer I/O during SYNC, from the point of view of slave. -# 2) Master timeout from the point of view of slaves (data, pings). -# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value -# specified for repl-ping-slave-period otherwise a timeout will be detected -# every time there is low traffic between the master and the slave. +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. # # repl-timeout 60 -# Disable TCP_NODELAY on the slave socket after SYNC? +# Disable TCP_NODELAY on the replica socket after SYNC? # # If you select "yes" Redis will use a smaller number of TCP packets and -# less bandwidth to send data to slaves. But this can add a delay for -# the data to appear on the slave side, up to 40 milliseconds with +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with # Linux kernels using a default configuration. # -# If you select "no" the delay for data to appear on the slave side will +# If you select "no" the delay for data to appear on the replica side will # be reduced but more bandwidth will be used for replication. # # By default we optimize for low latency, but in very high traffic conditions -# or when the master and slaves are many hops away, turning this to "yes" may +# or when the master and replicas are many hops away, turning this to "yes" may # be a good idea. repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# slave data when slaves are disconnected for some time, so that when a slave +# replica data when replicas are disconnected for some time, so that when a replica # wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the slave missed while +# resync is enough, just passing the portion of data the replica missed while # disconnected. # -# The biggest the replication backlog, the longer the time the slave can be +# The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a slave connected. +# The backlog is only allocated once there is at least a replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected slaves for some time, the backlog +# After a master has no longer connected replicas for some time, the backlog # will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last slave disconnected, for +# need to elapse, starting from the time the last replica disconnected, for # the backlog buffer to be freed. # +# Note that replicas never free the backlog for timeout, since they may be +# promoted to masters later, and should be able to correctly "partially +# resynchronize" with the replicas: hence they should always accumulate backlog. +# # A value of 0 means to never release the backlog. # # repl-backlog-ttl 3600 -# The slave priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a slave to promote into a +# The replica priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a replica to promote into a # master if the master is no longer working correctly. # -# A slave with a low priority number is considered better for promotion, so -# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel will # pick the one with priority 10, that is the lowest. # -# However a special priority of 0 marks the slave as not able to perform the -# role of master, so a slave with priority of 0 will never be selected by +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by # Redis Sentinel for promotion. # # By default the priority is 100. -slave-priority 100 +replica-priority 100 # It is possible for a master to stop accepting writes if there are less than -# N slaves connected, having a lag less or equal than M seconds. +# N replicas connected, having a lag less or equal than M seconds. # -# The N slaves need to be in "online" state. +# The N replicas need to be in "online" state. # # The lag in seconds, that must be <= the specified value, is calculated from -# the last ping received from the slave, that is usually sent every second. +# the last ping received from the replica, that is usually sent every second. # -# This option does not GUARANTEES that N replicas will accept the write, but -# will limit the window of exposure for lost writes in case not enough slaves +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough replicas # are available, to the specified number of seconds. # -# For example to require at least 3 slaves with a lag <= 10 seconds use: +# For example to require at least 3 replicas with a lag <= 10 seconds use: # -# min-slaves-to-write 3 -# min-slaves-max-lag 10 +# min-replicas-to-write 3 +# min-replicas-max-lag 10 # # Setting one or the other to 0 disables the feature. # -# By default min-slaves-to-write is set to 0 (feature disabled) and -# min-slaves-max-lag is set to 10. +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. + +# A Redis master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP and address normally reported by a replica is obtained +# in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may be actually reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 ################################## SECURITY ################################### @@ -355,9 +523,9 @@ slave-priority 100 # rename-command CONFIG "" # # Please note that changing the name of commands that are logged into the -# AOF file or transmitted to slaves may cause problems. +# AOF file or transmitted to replicas may cause problems. -################################### LIMITS #################################### +################################### CLIENTS #################################### # Set the max number of connected clients at the same time. By default # this limit is set to 10000 clients, however if the Redis server is not @@ -370,7 +538,9 @@ slave-priority 100 # # maxclients 10000 -# Don't use more memory than the specified amount of bytes. +############################## MEMORY MANAGEMENT ################################ + +# Set a memory usage limit to the specified amount of bytes. # When the memory limit is reached Redis will try to remove keys # according to the eviction policy selected (see maxmemory-policy). # @@ -379,18 +549,18 @@ slave-priority 100 # that would use more memory, like SET, LPUSH, and so on, and will continue # to reply to read-only commands like GET. # -# This option is usually useful when using Redis as an LRU cache, or to set -# a hard memory limit for an instance (using the 'noeviction' policy). +# This option is usually useful when using Redis as an LRU or LFU cache, or to +# set a hard memory limit for an instance (using the 'noeviction' policy). # -# WARNING: If you have slaves attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the slaves are subtracted +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted # from the used memory count, so that network problems / resyncs will # not trigger a loop where keys are evicted, and in turn the output -# buffer of slaves is full with DELs of keys evicted triggering the deletion +# buffer of replicas is full with DELs of keys evicted triggering the deletion # of more keys, and so forth until the database is completely emptied. # -# In short... if you have slaves attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for slave +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica # output buffers (but this is not needed if the policy is 'noeviction'). # # maxmemory @@ -398,17 +568,25 @@ slave-priority 100 # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory # is reached. You can select among five behaviors: # -# volatile-lru -> remove the key with an expire set using an LRU algorithm -# allkeys-lru -> remove any key accordingly to the LRU algorithm -# volatile-random -> remove a random key with an expire set -# allkeys-random -> remove a random key, any key -# volatile-ttl -> remove the key with the nearest expire time (minor TTL) -# noeviction -> don't expire at all, just return an error on write operations +# volatile-lru -> Evict using approximated LRU among the keys with an expire set. +# allkeys-lru -> Evict any key using approximated LRU. +# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. +# allkeys-lfu -> Evict any key using approximated LFU. +# volatile-random -> Remove a random key among the ones with an expire set. +# allkeys-random -> Remove a random key, any key. +# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) +# noeviction -> Don't evict anything, just return an error on write operations. +# +# LRU means Least Recently Used +# LFU means Least Frequently Used +# +# Both LRU, LFU and volatile-ttl are implemented using approximated +# randomized algorithms. # # Note: with any of the above policies, Redis will return an error on write -# operations, when there are not suitable keys for eviction. +# operations, when there are no suitable keys for eviction. # -# At the date of writing this commands are: set setnx setex append +# At the date of writing these commands are: set setnx setex append # incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd # sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby # zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby @@ -416,15 +594,87 @@ slave-priority 100 # # The default is: # -# maxmemory-policy volatile-lru +# maxmemory-policy noeviction -# LRU and minimal TTL algorithms are not precise algorithms but approximated -# algorithms (in order to save memory), so you can select as well the sample -# size to check. For instance for default Redis will check three keys and -# pick the one that was used less recently, you can change the sample size -# using the following configuration directive. +# LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can tune it for speed or +# accuracy. For default Redis will check five keys and pick the one that was +# used less recently, you can change the sample size using the following +# configuration directive. # -# maxmemory-samples 3 +# The default of 5 produces good enough results. 10 Approximates very closely +# true LRU but costs more CPU. 3 is faster but not very accurate. +# +# maxmemory-samples 5 + +# Starting from Redis 5, by default a replica will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the replica as keys evict in the master side. +# +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica to have +# a different memory setting, and you are sure all the writes performed to the +# replica are idempotent, then you may change this default (but be sure to understand +# what you are doing). +# +# Note that since the replica by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the replica, or data structures may sometimes take more memory and so +# forth). So make sure you monitor your replicas and make sure they have enough +# memory to never hit a real out-of-memory condition before the master hits +# the configured maxmemory setting. +# +# replica-ignore-maxmemory yes + +############################# LAZY FREEING #################################### + +# Redis has two primitives to delete keys. One is called DEL and is a blocking +# deletion of the object. It means that the server stops processing new commands +# in order to reclaim all the memory associated with an object in a synchronous +# way. If the key deleted is associated with a small object, the time needed +# in order to execute the DEL command is very small and comparable to most other +# O(1) or O(log_N) commands in Redis. However if the key is associated with an +# aggregated value containing millions of elements, the server can block for +# a long time (even seconds) in order to complete the operation. +# +# For the above reasons Redis also offers non blocking deletion primitives +# such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and +# FLUSHDB commands, in order to reclaim memory in background. Those commands +# are executed in constant time. Another thread will incrementally free the +# object in the background as fast as possible. +# +# DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. +# It's up to the design of the application to understand when it is a good +# idea to use one or the other. However the Redis server sometimes has to +# delete keys or flush the whole database as a side effect of other operations. +# Specifically Redis deletes objects independently of a user call in the +# following scenarios: +# +# 1) On eviction, because of the maxmemory and maxmemory policy configurations, +# in order to make room for new data, without going over the specified +# memory limit. +# 2) Because of expire: when a key with an associated time to live (see the +# EXPIRE command) must be deleted from memory. +# 3) Because of a side effect of a command that stores data on a key that may +# already exist. For example the RENAME command may delete the old key +# content when it is replaced with another one. Similarly SUNIONSTORE +# or SORT with STORE option may delete existing keys. The SET command +# itself removes any old content of the specified key in order to replace +# it with the specified string. +# 4) During replication, when a replica performs a full resynchronization with +# its master, the content of the whole database is removed in order to +# load the RDB file just transferred. +# +# In all the above cases the default is to delete objects in a blocking way, +# like if DEL was called. However you can configure each case specifically +# in order to instead release memory in a non-blocking way like if UNLINK +# was called, using the following configuration directives: + +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no ############################## APPEND ONLY MODE ############################### @@ -453,13 +703,13 @@ appendonly no appendfilename "appendonly.aof" # The fsync() call tells the Operating System to actually write data on disk -# instead to wait for more data in the output buffer. Some OS will really flush +# instead of waiting for more data in the output buffer. Some OS will really flush # data on disk, some other OS will just try to do it ASAP. # # Redis supports three different modes: # # no: don't fsync, just let the OS flush the data when it wants. Faster. -# always: fsync after every write to the append only log . Slow, Safest. +# always: fsync after every write to the append only log. Slow, Safest. # everysec: fsync only one time every second. Compromise. # # The default is "everysec", as that's usually the right compromise between @@ -520,6 +770,41 @@ no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +# When rewriting the AOF file, Redis is able to use an RDB preamble in the +# AOF file for faster rewrites and recoveries. When this option is turned +# on the rewritten AOF file is composed of two different stanzas: +# +# [RDB file][AOF tail] +# +# When loading Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, and continues loading the AOF +# tail. +aof-use-rdb-preamble yes + ################################ LUA SCRIPTING ############################### # Max execution time of a Lua script in milliseconds. @@ -528,16 +813,167 @@ auto-aof-rewrite-min-size 64mb # still in execution after the maximum allowed time and will start to # reply to queries with an error. # -# When a long running script exceed the maximum execution time only the +# When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be # used to stop a script that did not yet called write commands. The second -# is the only way to shut down the server in the case a write commands was -# already issue by the script but the user don't want to wait for the natural +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural # termination of the script. # # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 +################################ REDIS CLUSTER ############################### +# +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however +# in order to mark it as "mature" we need to wait for a non trivial percentage +# of users to deploy it in production. +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system do not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# A replica of a failing master will avoid to start a failover if its data +# looks too old. +# +# There is no simple way for a replica to actually have an exact measure of +# its "data age", so the following two checks are performed: +# +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best +# replication offset (more data from the master processed). +# Replicas will try to get their rank by offset, and apply to the start +# of the failover a delay proportional to their rank. +# +# 2) Every single replica computes the time of the last interaction with +# its master. This can be the last ping or command received (if the master +# is still in the "connected" state), or the time that elapsed since the +# disconnection with the master (if the replication link is currently down). +# If the last interaction is too old, the replica will not try to failover +# at all. +# +# The point "2" can be tuned by user. Specifically a replica will not perform +# the failover if, since the last interaction with the master, the time +# elapsed is greater than: +# +# (node-timeout * replica-validity-factor) + repl-ping-replica-period +# +# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master +# for longer than 310 seconds. +# +# A large replica-validity-factor may allow replicas with too old data to failover +# a master, while a too small value may prevent the cluster from being able to +# elect a replica at all. +# +# For maximum availability, it is possible to set the replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the +# master regardless of the last time they interacted with the master. +# (However they'll always try to apply a delay proportional to their +# offset rank). +# +# Zero is the only value able to guarantee that when all the partitions heal +# the cluster will always be able to continue. +# +# cluster-replica-validity-factor 10 + +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability +# to resist to failures as otherwise an orphaned master can't be failed over +# in case of failure if it has no working replicas. +# +# Replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every +# master in your cluster. +# +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value. +# A value of 0 can be set but is useful only for debugging and dangerous +# in production. +# +# cluster-migration-barrier 1 + +# By default Redis Cluster nodes stop accepting queries if they detect there +# is at least an hash slot uncovered (no available node is serving it). +# This way if the cluster is partially down (for example a range of hash slots +# are no longer covered) all the cluster becomes, eventually, unavailable. +# It automatically returns available as soon as all the slots are covered again. +# +# However sometimes you want the subset of the cluster which is working, +# to continue to accept queries for the part of the key space that is still +# covered. In order to do so, just set the cluster-require-full-coverage +# option to no. +# +# cluster-require-full-coverage yes + +# This option, when set to yes, prevents replicas from trying to failover its +# master during master failures. However the master can still perform a +# manual failover, if forced to do so. +# +# This is useful in different scenarios, especially in the case of multiple +# data center operations, where we want one side to never be promoted if not +# in the case of a total DC failure. +# +# cluster-replica-no-failover no + +# In order to setup your cluster make sure to read the documentation +# available at http://redis.io web site. + +########################## CLUSTER DOCKER/NAT support ######################## + +# In certain deployments, Redis Cluster nodes address discovery fails, because +# addresses are NAT-ted or because ports are forwarded (the typical case is +# Docker and other containers). +# +# In order to make Redis Cluster working in such environments, a static +# configuration where each node knows its public address is needed. The +# following two options are used for this scope, and are: +# +# * cluster-announce-ip +# * cluster-announce-port +# * cluster-announce-bus-port +# +# Each instruct the node about its address, client port, and cluster message +# bus port. The information is then published in the header of the bus packets +# so that other nodes will be able to correctly map the address of the node +# publishing the information. +# +# If the above options are not used, the normal Redis Cluster auto-detection +# will be used instead. +# +# Note that when remapped, the bus port may not be at the fixed offset of +# clients port + 10000, so you can specify any port and bus-port depending +# on how they get remapped. If the bus-port is not set, a fixed offset of +# 10000 will be used as usually. +# +# Example: +# +# cluster-announce-ip 10.1.1.5 +# cluster-announce-port 6379 +# cluster-announce-bus-port 6380 + ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified @@ -562,10 +998,31 @@ slowlog-log-slower-than 10000 # You can reclaim memory used by the slow log with SLOWLOG RESET. slowlog-max-len 128 -############################# Event notification ############################## +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/keyspace-events +# This feature is documented at http://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -590,8 +1047,8 @@ slowlog-max-len 128 # A Alias for g$lshzxe, so that the "AKE" string means all the events. # # The "notify-keyspace-events" takes as argument a string that is composed -# by zero or multiple characters. The empty string means that notifications -# are disabled at all. +# of zero or multiple characters. The empty string means that notifications +# are disabled. # # Example: to enable list and generic events, from the point of view of the # event name, use: @@ -616,14 +1073,39 @@ notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 # Sets have a special encoding in just one case: when a set is composed -# of just strings that happens to be integers in radix 10 in the range +# of just strings that happen to be integers in radix 10 in the range # of 64 bit signed integers. # The following configuration setting sets the limit in the size of the # set in order to use this special memory saving encoding. @@ -635,6 +1117,31 @@ set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Streams macro node max size / items. The stream data structure is a radix +# tree of big nodes that encode multiple items inside. Using this configuration +# it is possible to configure how big a single node can be in bytes, and the +# maximum number of items it may contain before switching to a new node when +# appending new stream entries. If any of the following settings are set to +# zero, the limit is ignored, so for instance it is possible to set just a +# max entires limit by setting max-bytes to 0 and max-entries to the desired +# value. +stream-node-max-bytes 4096 +stream-node-max-entries 100 + # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level # keys to values). The hash table implementation Redis uses (see dict.c) @@ -644,11 +1151,11 @@ zset-max-ziplist-value 64 # by the hash table. # # The default is to use this millisecond 10 times every second in order to -# active rehashing the main dictionaries, freeing memory when possible. +# actively rehash the main dictionaries, freeing memory when possible. # # If unsure: # use "activerehashing no" if you have hard latency requirements and it is -# not a good thing in your environment that Redis can reply form time to time +# not a good thing in your environment that Redis can reply from time to time # to queries with 2 milliseconds delay. # # use "activerehashing yes" if you don't have such hard requirements but @@ -662,8 +1169,8 @@ activerehashing yes # # The limit can be set differently for the three different classes of clients: # -# normal -> normal clients -# slave -> slave clients and MONITOR clients +# normal -> normal clients including MONITOR clients +# replica -> replica clients # pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: @@ -684,20 +1191,34 @@ activerehashing yes # asynchronous clients may create a scenario where data is requested faster # than it can read. # -# Instead there is a default limit for pubsub and slave clients, since -# subscribers and slaves receive data in a push fashion. +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. # # Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit replica 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default in order to avoid that a protocol desynchronization (for +# instance due to a bug in the client) will lead to unbound memory usage in +# the query buffer. However you can configure it here if you have very special +# needs, such us huge multi/exec requests or alike. +# +# client-query-buffer-limit 1gb + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited ot 512 mb. However you can change this limit +# here. +# +# proto-max-bulk-len 512mb + # Redis calls an internal function to perform many background tasks, like # closing connections of clients in timeout, purging expired keys that are # never requested, and so forth. # # Not all tasks are performed with the same frequency, but Redis checks for -# tasks to perform accordingly to the specified "hz" value. +# tasks to perform according to the specified "hz" value. # # By default "hz" is set to 10. Raising the value will use more CPU when # Redis is idle, but at the same time will make Redis more responsive when @@ -709,9 +1230,148 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # 100 only in environments where very low latency is required. hz 10 +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporary raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used as +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + # When a child rewrites the AOF file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. aof-rewrite-incremental-fsync yes +# When redis saves RDB file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +rdb-save-incremental-fsync yes + +# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good +# idea to start with the default settings and only change them after investigating +# how to improve the performances and how the keys LFU change over time, which +# is possible to inspect via the OBJECT FREQ command. +# +# There are two tunable parameters in the Redis LFU implementation: the +# counter logarithm factor and the counter decay time. It is important to +# understand what the two parameters mean before changing them. +# +# The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis +# uses a probabilistic increment with logarithmic behavior. Given the value +# of the old counter, when a key is accessed, the counter is incremented in +# this way: +# +# 1. A random number R between 0 and 1 is extracted. +# 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). +# 3. The counter is incremented only if R < P. +# +# The default lfu-log-factor is 10. This is a table of how the frequency +# counter changes with a different number of accesses with different +# logarithmic factors: +# +# +--------+------------+------------+------------+------------+------------+ +# | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | +# +--------+------------+------------+------------+------------+------------+ +# | 0 | 104 | 255 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 1 | 18 | 49 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 10 | 10 | 18 | 142 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 100 | 8 | 11 | 49 | 143 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# +# NOTE: The above table was obtained by running the following commands: +# +# redis-benchmark -n 1000000 incr foo +# redis-cli object freq foo +# +# NOTE 2: The counter initial value is 5 in order to give new objects a chance +# to accumulate hits. +# +# The counter decay time is the time, in minutes, that must elapse in order +# for the key counter to be divided by two (or decremented if it has a value +# less <= 10). +# +# The default value for the lfu-decay-time is 1. A Special value of 0 means to +# decay the counter every time it happens to be scanned. +# +# lfu-log-factor 10 +# lfu-decay-time 1 + +########################### ACTIVE DEFRAGMENTATION ####################### +# +# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested +# even in production and manually tested by multiple engineers for some +# time. +# +# What is active defragmentation? +# ------------------------------- +# +# Active (online) defragmentation allows a Redis server to compact the +# spaces left between small allocations and deallocations of data in memory, +# thus allowing to reclaim back memory. +# +# Fragmentation is a natural process that happens with every allocator (but +# less so with Jemalloc, fortunately) and certain workloads. Normally a server +# restart is needed in order to lower the fragmentation, or at least to flush +# away all the data and create it again. However thanks to this feature +# implemented by Oran Agra for Redis 4.0 this process can happen at runtime +# in an "hot" way, while the server is running. +# +# Basically when the fragmentation is over a certain level (see the +# configuration options below) Redis will start to create new copies of the +# values in contiguous memory regions by exploiting certain specific Jemalloc +# features (in order to understand if an allocation is causing fragmentation +# and to allocate it in a better place), and at the same time, will release the +# old copies of the data. This process, repeated incrementally for all the keys +# will cause the fragmentation to drop back to normal values. +# +# Important things to understand: +# +# 1. This feature is disabled by default, and only works if you compiled Redis +# to use the copy of Jemalloc we ship with the source code of Redis. +# This is the default with Linux builds. +# +# 2. You never need to enable this feature if you don't have fragmentation +# issues. +# +# 3. Once you experience fragmentation, you can enable this feature when +# needed with the command "CONFIG SET activedefrag yes". +# +# The configuration parameters are able to fine tune the behavior of the +# defragmentation process. If you are not sure about what they mean it is +# a good idea to leave the defaults untouched. + +# Enabled active defragmentation +# activedefrag yes + +# Minimum amount of fragmentation waste to start active defrag +# active-defrag-ignore-bytes 100mb + +# Minimum percentage of fragmentation to start active defrag +# active-defrag-threshold-lower 10 + +# Maximum percentage of fragmentation at which we use maximum effort +# active-defrag-threshold-upper 100 + +# Minimal effort for defrag in CPU percentage +# active-defrag-cycle-min 5 + +# Maximal effort for defrag in CPU percentage +# active-defrag-cycle-max 75 + +# Maximum number of set/hash/zset/list fields that will be processed from +# the main dictionary scan +# active-defrag-max-scan-fields 1000 diff --git a/server/configs/6380.conf b/server/configs/6380.conf index db3234f..b225ebd 100644 --- a/server/configs/6380.conf +++ b/server/configs/6380.conf @@ -1,4 +1,9 @@ -# Redis configuration file example +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: @@ -15,7 +20,7 @@ ################################## INCLUDES ################################### # Include one or more other config files here. This is useful if you -# have a standard template that goes to all Redis server but also need +# have a standard template that goes to all Redis servers but also need # to customize a few per-server settings. Include files can include # other files, so use this wisely. # @@ -30,17 +35,59 @@ # include /path/to/local.conf # include /path/to/other.conf -################################ GENERAL ##################################### +################################## MODULES ##################################### -# By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize no +# Load modules at startup. If the server is not able to load modules +# it will abort. It is possible to use multiple loadmodule directives. +# +# loadmodule /path/to/my_module.so +# loadmodule /path/to/other_module.so -# When running daemonized, Redis writes a pid file in /var/run/redis.pid by -# default. You can specify a custom pid file location here. -#pidfile /var/run/redis.pid +################################## NETWORK ##################################### -# Accept connections on the specified port, default is 6379. +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all the network interfaces available on the server. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 ::1 +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only into +# the IPv4 loopback interface address (this means Redis will be able to +# accept connections only from clients running into the same computer it +# is running). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +bind 127.0.0.1 + +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode yes + +# Accept connections on the specified port, default is 6379 (IANA #815344). # If port 0 is specified Redis will not listen on a TCP socket. port 6380 @@ -53,22 +100,14 @@ port 6380 # in order to get the desired effect. tcp-backlog 511 -# By default Redis listens for connections from all the network interfaces -# available on the server. It is possible to listen to just one or multiple -# interfaces using the "bind" configuration directive, followed by one or -# more IP addresses. +# Unix socket. # -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 - # Specify the path for the Unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # -#unixsocket /tmp/redis.sock -#unixsocketperm 755 +# unixsocket /tmp/redis.sock +# unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) timeout 0 @@ -86,8 +125,37 @@ timeout 0 # Note that to close the connection the double of the time is needed. # On other kernels the period depends on the kernel configuration. # -# A reasonable value for this option is 60 seconds. -tcp-keepalive 0 +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize no + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised no + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +pidfile /var/run/redis_6379.pid # Specify the server verbosity level. # This can be one of: @@ -117,6 +185,14 @@ logfile "" # dbid is a number between 0 and 'databases'-1 databases 16 +# By default Redis shows an ASCII art logo only when started to log to the +# standard output and if the standard output is a TTY. Basically this means +# that normally a logo is displayed only in interactive sessions. +# +# However it is possible to force the pre-4.0 behavior and always show a +# ASCII art logo in startup logs by setting the following option to yes. +always-show-logo yes + ################################ SNAPSHOTTING ################################ # # Save the DB on disk: @@ -131,7 +207,7 @@ databases 16 # after 300 sec (5 min) if at least 10 keys changed # after 60 sec if at least 10000 keys changed # -# Note: you can disable saving at all commenting all the "save" lines. +# Note: you can disable saving completely by commenting out all "save" lines. # # It is also possible to remove all the previously configured save # points by adding a save directive with a single empty string argument @@ -139,9 +215,9 @@ databases 16 # # save "" -#save 900 1 +save 900 1 #save 300 10 -save 7200 1 +save 60 1000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. @@ -174,7 +250,7 @@ rdbcompression yes rdbchecksum yes # The filename where to dump the DB -dbfilename dump6880.rdb +dbfilename dump6380.rdb # The working directory. # @@ -188,140 +264,232 @@ dir ./ ################################# REPLICATION ################################# -# Master-Slave replication. Use slaveof to make a Redis instance a copy of -# another Redis server. Note that the configuration is local to the slave -# so for example it is possible to configure the slave to save the DB with a -# different interval, or to listen to another port, and so on. +# Master-Replica replication. Use replicaof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. # -# slaveof +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition replicas automatically try to reconnect to masters +# and resynchronize with them. +# +# replicaof # If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before +# directive below) it is possible to tell the replica to authenticate before # starting the replication synchronization process, otherwise the master will -# refuse the slave request. +# refuse the replica request. # # masterauth -# When a slave loses its connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: # -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# 2) if replica-serve-stale-data is set to 'no' the replica will reply with # an error "SYNC with master in progress" to all the kind of commands -# but to INFO and SLAVEOF. +# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, +# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, +# COMMAND, POST, HOST: and LATENCY. # -slave-serve-stale-data yes +replica-serve-stale-data yes -# You can configure a slave instance to accept writes or not. Writing against -# a slave instance may be useful to store some ephemeral data (because data -# written on a slave will be easily deleted after resync with the master) but +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but # may also cause problems if clients are writing to it because of a # misconfiguration. # -# Since Redis 2.6 by default slaves are read-only. +# Since Redis 2.6 by default replicas are read-only. # -# Note: read only slaves are not designed to be exposed to untrusted clients +# Note: read only replicas are not designed to be exposed to untrusted clients # on the internet. It's just a protection layer against misuse of the instance. -# Still a read only slave exports by default all the administrative commands +# Still a read only replica exports by default all the administrative commands # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve -# security of read only slaves using 'rename-command' to shadow all the +# security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. -slave-read-only yes +replica-read-only yes -# Slaves send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_slave_period option. The default value is 10 +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New replicas and reconnecting replicas that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the replicas. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the replicas incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to replica sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more replicas +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new replicas arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple replicas +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that transfers the RDB via socket +# to the replicas. +# +# This is important since once the transfer starts, it is not possible to serve +# new replicas arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more replicas arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Replicas send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_replica_period option. The default value is 10 # seconds. # -# repl-ping-slave-period 10 +# repl-ping-replica-period 10 # The following option sets the replication timeout for: # -# 1) Bulk transfer I/O during SYNC, from the point of view of slave. -# 2) Master timeout from the point of view of slaves (data, pings). -# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value -# specified for repl-ping-slave-period otherwise a timeout will be detected -# every time there is low traffic between the master and the slave. +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. # # repl-timeout 60 -# Disable TCP_NODELAY on the slave socket after SYNC? +# Disable TCP_NODELAY on the replica socket after SYNC? # # If you select "yes" Redis will use a smaller number of TCP packets and -# less bandwidth to send data to slaves. But this can add a delay for -# the data to appear on the slave side, up to 40 milliseconds with +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with # Linux kernels using a default configuration. # -# If you select "no" the delay for data to appear on the slave side will +# If you select "no" the delay for data to appear on the replica side will # be reduced but more bandwidth will be used for replication. # # By default we optimize for low latency, but in very high traffic conditions -# or when the master and slaves are many hops away, turning this to "yes" may +# or when the master and replicas are many hops away, turning this to "yes" may # be a good idea. repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# slave data when slaves are disconnected for some time, so that when a slave +# replica data when replicas are disconnected for some time, so that when a replica # wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the slave missed while +# resync is enough, just passing the portion of data the replica missed while # disconnected. # -# The biggest the replication backlog, the longer the time the slave can be +# The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a slave connected. +# The backlog is only allocated once there is at least a replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected slaves for some time, the backlog +# After a master has no longer connected replicas for some time, the backlog # will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last slave disconnected, for +# need to elapse, starting from the time the last replica disconnected, for # the backlog buffer to be freed. # +# Note that replicas never free the backlog for timeout, since they may be +# promoted to masters later, and should be able to correctly "partially +# resynchronize" with the replicas: hence they should always accumulate backlog. +# # A value of 0 means to never release the backlog. # # repl-backlog-ttl 3600 -# The slave priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a slave to promote into a +# The replica priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a replica to promote into a # master if the master is no longer working correctly. # -# A slave with a low priority number is considered better for promotion, so -# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel will # pick the one with priority 10, that is the lowest. # -# However a special priority of 0 marks the slave as not able to perform the -# role of master, so a slave with priority of 0 will never be selected by +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by # Redis Sentinel for promotion. # # By default the priority is 100. -slave-priority 100 +replica-priority 100 # It is possible for a master to stop accepting writes if there are less than -# N slaves connected, having a lag less or equal than M seconds. +# N replicas connected, having a lag less or equal than M seconds. # -# The N slaves need to be in "online" state. +# The N replicas need to be in "online" state. # # The lag in seconds, that must be <= the specified value, is calculated from -# the last ping received from the slave, that is usually sent every second. +# the last ping received from the replica, that is usually sent every second. # -# This option does not GUARANTEES that N replicas will accept the write, but -# will limit the window of exposure for lost writes in case not enough slaves +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough replicas # are available, to the specified number of seconds. # -# For example to require at least 3 slaves with a lag <= 10 seconds use: +# For example to require at least 3 replicas with a lag <= 10 seconds use: # -# min-slaves-to-write 3 -# min-slaves-max-lag 10 +# min-replicas-to-write 3 +# min-replicas-max-lag 10 # # Setting one or the other to 0 disables the feature. # -# By default min-slaves-to-write is set to 0 (feature disabled) and -# min-slaves-max-lag is set to 10. +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. + +# A Redis master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP and address normally reported by a replica is obtained +# in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may be actually reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 ################################## SECURITY ################################### @@ -355,9 +523,9 @@ slave-priority 100 # rename-command CONFIG "" # # Please note that changing the name of commands that are logged into the -# AOF file or transmitted to slaves may cause problems. +# AOF file or transmitted to replicas may cause problems. -################################### LIMITS #################################### +################################### CLIENTS #################################### # Set the max number of connected clients at the same time. By default # this limit is set to 10000 clients, however if the Redis server is not @@ -370,7 +538,9 @@ slave-priority 100 # # maxclients 10000 -# Don't use more memory than the specified amount of bytes. +############################## MEMORY MANAGEMENT ################################ + +# Set a memory usage limit to the specified amount of bytes. # When the memory limit is reached Redis will try to remove keys # according to the eviction policy selected (see maxmemory-policy). # @@ -379,18 +549,18 @@ slave-priority 100 # that would use more memory, like SET, LPUSH, and so on, and will continue # to reply to read-only commands like GET. # -# This option is usually useful when using Redis as an LRU cache, or to set -# a hard memory limit for an instance (using the 'noeviction' policy). +# This option is usually useful when using Redis as an LRU or LFU cache, or to +# set a hard memory limit for an instance (using the 'noeviction' policy). # -# WARNING: If you have slaves attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the slaves are subtracted +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted # from the used memory count, so that network problems / resyncs will # not trigger a loop where keys are evicted, and in turn the output -# buffer of slaves is full with DELs of keys evicted triggering the deletion +# buffer of replicas is full with DELs of keys evicted triggering the deletion # of more keys, and so forth until the database is completely emptied. # -# In short... if you have slaves attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for slave +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica # output buffers (but this is not needed if the policy is 'noeviction'). # # maxmemory @@ -398,17 +568,25 @@ slave-priority 100 # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory # is reached. You can select among five behaviors: # -# volatile-lru -> remove the key with an expire set using an LRU algorithm -# allkeys-lru -> remove any key accordingly to the LRU algorithm -# volatile-random -> remove a random key with an expire set -# allkeys-random -> remove a random key, any key -# volatile-ttl -> remove the key with the nearest expire time (minor TTL) -# noeviction -> don't expire at all, just return an error on write operations +# volatile-lru -> Evict using approximated LRU among the keys with an expire set. +# allkeys-lru -> Evict any key using approximated LRU. +# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. +# allkeys-lfu -> Evict any key using approximated LFU. +# volatile-random -> Remove a random key among the ones with an expire set. +# allkeys-random -> Remove a random key, any key. +# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) +# noeviction -> Don't evict anything, just return an error on write operations. +# +# LRU means Least Recently Used +# LFU means Least Frequently Used +# +# Both LRU, LFU and volatile-ttl are implemented using approximated +# randomized algorithms. # # Note: with any of the above policies, Redis will return an error on write -# operations, when there are not suitable keys for eviction. +# operations, when there are no suitable keys for eviction. # -# At the date of writing this commands are: set setnx setex append +# At the date of writing these commands are: set setnx setex append # incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd # sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby # zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby @@ -416,15 +594,87 @@ slave-priority 100 # # The default is: # -# maxmemory-policy volatile-lru +# maxmemory-policy noeviction -# LRU and minimal TTL algorithms are not precise algorithms but approximated -# algorithms (in order to save memory), so you can select as well the sample -# size to check. For instance for default Redis will check three keys and -# pick the one that was used less recently, you can change the sample size -# using the following configuration directive. +# LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can tune it for speed or +# accuracy. For default Redis will check five keys and pick the one that was +# used less recently, you can change the sample size using the following +# configuration directive. # -# maxmemory-samples 3 +# The default of 5 produces good enough results. 10 Approximates very closely +# true LRU but costs more CPU. 3 is faster but not very accurate. +# +# maxmemory-samples 5 + +# Starting from Redis 5, by default a replica will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the replica as keys evict in the master side. +# +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica to have +# a different memory setting, and you are sure all the writes performed to the +# replica are idempotent, then you may change this default (but be sure to understand +# what you are doing). +# +# Note that since the replica by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the replica, or data structures may sometimes take more memory and so +# forth). So make sure you monitor your replicas and make sure they have enough +# memory to never hit a real out-of-memory condition before the master hits +# the configured maxmemory setting. +# +# replica-ignore-maxmemory yes + +############################# LAZY FREEING #################################### + +# Redis has two primitives to delete keys. One is called DEL and is a blocking +# deletion of the object. It means that the server stops processing new commands +# in order to reclaim all the memory associated with an object in a synchronous +# way. If the key deleted is associated with a small object, the time needed +# in order to execute the DEL command is very small and comparable to most other +# O(1) or O(log_N) commands in Redis. However if the key is associated with an +# aggregated value containing millions of elements, the server can block for +# a long time (even seconds) in order to complete the operation. +# +# For the above reasons Redis also offers non blocking deletion primitives +# such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and +# FLUSHDB commands, in order to reclaim memory in background. Those commands +# are executed in constant time. Another thread will incrementally free the +# object in the background as fast as possible. +# +# DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. +# It's up to the design of the application to understand when it is a good +# idea to use one or the other. However the Redis server sometimes has to +# delete keys or flush the whole database as a side effect of other operations. +# Specifically Redis deletes objects independently of a user call in the +# following scenarios: +# +# 1) On eviction, because of the maxmemory and maxmemory policy configurations, +# in order to make room for new data, without going over the specified +# memory limit. +# 2) Because of expire: when a key with an associated time to live (see the +# EXPIRE command) must be deleted from memory. +# 3) Because of a side effect of a command that stores data on a key that may +# already exist. For example the RENAME command may delete the old key +# content when it is replaced with another one. Similarly SUNIONSTORE +# or SORT with STORE option may delete existing keys. The SET command +# itself removes any old content of the specified key in order to replace +# it with the specified string. +# 4) During replication, when a replica performs a full resynchronization with +# its master, the content of the whole database is removed in order to +# load the RDB file just transferred. +# +# In all the above cases the default is to delete objects in a blocking way, +# like if DEL was called. However you can configure each case specifically +# in order to instead release memory in a non-blocking way like if UNLINK +# was called, using the following configuration directives: + +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no ############################## APPEND ONLY MODE ############################### @@ -453,13 +703,13 @@ appendonly no appendfilename "appendonly.aof" # The fsync() call tells the Operating System to actually write data on disk -# instead to wait for more data in the output buffer. Some OS will really flush +# instead of waiting for more data in the output buffer. Some OS will really flush # data on disk, some other OS will just try to do it ASAP. # # Redis supports three different modes: # # no: don't fsync, just let the OS flush the data when it wants. Faster. -# always: fsync after every write to the append only log . Slow, Safest. +# always: fsync after every write to the append only log. Slow, Safest. # everysec: fsync only one time every second. Compromise. # # The default is "everysec", as that's usually the right compromise between @@ -520,6 +770,41 @@ no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +# When rewriting the AOF file, Redis is able to use an RDB preamble in the +# AOF file for faster rewrites and recoveries. When this option is turned +# on the rewritten AOF file is composed of two different stanzas: +# +# [RDB file][AOF tail] +# +# When loading Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, and continues loading the AOF +# tail. +aof-use-rdb-preamble yes + ################################ LUA SCRIPTING ############################### # Max execution time of a Lua script in milliseconds. @@ -528,16 +813,167 @@ auto-aof-rewrite-min-size 64mb # still in execution after the maximum allowed time and will start to # reply to queries with an error. # -# When a long running script exceed the maximum execution time only the +# When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be # used to stop a script that did not yet called write commands. The second -# is the only way to shut down the server in the case a write commands was -# already issue by the script but the user don't want to wait for the natural +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural # termination of the script. # # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 +################################ REDIS CLUSTER ############################### +# +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however +# in order to mark it as "mature" we need to wait for a non trivial percentage +# of users to deploy it in production. +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system do not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# A replica of a failing master will avoid to start a failover if its data +# looks too old. +# +# There is no simple way for a replica to actually have an exact measure of +# its "data age", so the following two checks are performed: +# +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best +# replication offset (more data from the master processed). +# Replicas will try to get their rank by offset, and apply to the start +# of the failover a delay proportional to their rank. +# +# 2) Every single replica computes the time of the last interaction with +# its master. This can be the last ping or command received (if the master +# is still in the "connected" state), or the time that elapsed since the +# disconnection with the master (if the replication link is currently down). +# If the last interaction is too old, the replica will not try to failover +# at all. +# +# The point "2" can be tuned by user. Specifically a replica will not perform +# the failover if, since the last interaction with the master, the time +# elapsed is greater than: +# +# (node-timeout * replica-validity-factor) + repl-ping-replica-period +# +# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master +# for longer than 310 seconds. +# +# A large replica-validity-factor may allow replicas with too old data to failover +# a master, while a too small value may prevent the cluster from being able to +# elect a replica at all. +# +# For maximum availability, it is possible to set the replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the +# master regardless of the last time they interacted with the master. +# (However they'll always try to apply a delay proportional to their +# offset rank). +# +# Zero is the only value able to guarantee that when all the partitions heal +# the cluster will always be able to continue. +# +# cluster-replica-validity-factor 10 + +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability +# to resist to failures as otherwise an orphaned master can't be failed over +# in case of failure if it has no working replicas. +# +# Replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every +# master in your cluster. +# +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value. +# A value of 0 can be set but is useful only for debugging and dangerous +# in production. +# +# cluster-migration-barrier 1 + +# By default Redis Cluster nodes stop accepting queries if they detect there +# is at least an hash slot uncovered (no available node is serving it). +# This way if the cluster is partially down (for example a range of hash slots +# are no longer covered) all the cluster becomes, eventually, unavailable. +# It automatically returns available as soon as all the slots are covered again. +# +# However sometimes you want the subset of the cluster which is working, +# to continue to accept queries for the part of the key space that is still +# covered. In order to do so, just set the cluster-require-full-coverage +# option to no. +# +# cluster-require-full-coverage yes + +# This option, when set to yes, prevents replicas from trying to failover its +# master during master failures. However the master can still perform a +# manual failover, if forced to do so. +# +# This is useful in different scenarios, especially in the case of multiple +# data center operations, where we want one side to never be promoted if not +# in the case of a total DC failure. +# +# cluster-replica-no-failover no + +# In order to setup your cluster make sure to read the documentation +# available at http://redis.io web site. + +########################## CLUSTER DOCKER/NAT support ######################## + +# In certain deployments, Redis Cluster nodes address discovery fails, because +# addresses are NAT-ted or because ports are forwarded (the typical case is +# Docker and other containers). +# +# In order to make Redis Cluster working in such environments, a static +# configuration where each node knows its public address is needed. The +# following two options are used for this scope, and are: +# +# * cluster-announce-ip +# * cluster-announce-port +# * cluster-announce-bus-port +# +# Each instruct the node about its address, client port, and cluster message +# bus port. The information is then published in the header of the bus packets +# so that other nodes will be able to correctly map the address of the node +# publishing the information. +# +# If the above options are not used, the normal Redis Cluster auto-detection +# will be used instead. +# +# Note that when remapped, the bus port may not be at the fixed offset of +# clients port + 10000, so you can specify any port and bus-port depending +# on how they get remapped. If the bus-port is not set, a fixed offset of +# 10000 will be used as usually. +# +# Example: +# +# cluster-announce-ip 10.1.1.5 +# cluster-announce-port 6379 +# cluster-announce-bus-port 6380 + ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified @@ -562,10 +998,31 @@ slowlog-log-slower-than 10000 # You can reclaim memory used by the slow log with SLOWLOG RESET. slowlog-max-len 128 -############################# Event notification ############################## +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/keyspace-events +# This feature is documented at http://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -590,8 +1047,8 @@ slowlog-max-len 128 # A Alias for g$lshzxe, so that the "AKE" string means all the events. # # The "notify-keyspace-events" takes as argument a string that is composed -# by zero or multiple characters. The empty string means that notifications -# are disabled at all. +# of zero or multiple characters. The empty string means that notifications +# are disabled. # # Example: to enable list and generic events, from the point of view of the # event name, use: @@ -616,14 +1073,39 @@ notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 # Sets have a special encoding in just one case: when a set is composed -# of just strings that happens to be integers in radix 10 in the range +# of just strings that happen to be integers in radix 10 in the range # of 64 bit signed integers. # The following configuration setting sets the limit in the size of the # set in order to use this special memory saving encoding. @@ -635,6 +1117,31 @@ set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Streams macro node max size / items. The stream data structure is a radix +# tree of big nodes that encode multiple items inside. Using this configuration +# it is possible to configure how big a single node can be in bytes, and the +# maximum number of items it may contain before switching to a new node when +# appending new stream entries. If any of the following settings are set to +# zero, the limit is ignored, so for instance it is possible to set just a +# max entires limit by setting max-bytes to 0 and max-entries to the desired +# value. +stream-node-max-bytes 4096 +stream-node-max-entries 100 + # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level # keys to values). The hash table implementation Redis uses (see dict.c) @@ -644,11 +1151,11 @@ zset-max-ziplist-value 64 # by the hash table. # # The default is to use this millisecond 10 times every second in order to -# active rehashing the main dictionaries, freeing memory when possible. +# actively rehash the main dictionaries, freeing memory when possible. # # If unsure: # use "activerehashing no" if you have hard latency requirements and it is -# not a good thing in your environment that Redis can reply form time to time +# not a good thing in your environment that Redis can reply from time to time # to queries with 2 milliseconds delay. # # use "activerehashing yes" if you don't have such hard requirements but @@ -662,8 +1169,8 @@ activerehashing yes # # The limit can be set differently for the three different classes of clients: # -# normal -> normal clients -# slave -> slave clients and MONITOR clients +# normal -> normal clients including MONITOR clients +# replica -> replica clients # pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: @@ -684,20 +1191,34 @@ activerehashing yes # asynchronous clients may create a scenario where data is requested faster # than it can read. # -# Instead there is a default limit for pubsub and slave clients, since -# subscribers and slaves receive data in a push fashion. +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. # # Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit replica 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default in order to avoid that a protocol desynchronization (for +# instance due to a bug in the client) will lead to unbound memory usage in +# the query buffer. However you can configure it here if you have very special +# needs, such us huge multi/exec requests or alike. +# +# client-query-buffer-limit 1gb + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited ot 512 mb. However you can change this limit +# here. +# +# proto-max-bulk-len 512mb + # Redis calls an internal function to perform many background tasks, like # closing connections of clients in timeout, purging expired keys that are # never requested, and so forth. # # Not all tasks are performed with the same frequency, but Redis checks for -# tasks to perform accordingly to the specified "hz" value. +# tasks to perform according to the specified "hz" value. # # By default "hz" is set to 10. Raising the value will use more CPU when # Redis is idle, but at the same time will make Redis more responsive when @@ -709,9 +1230,148 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # 100 only in environments where very low latency is required. hz 10 +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporary raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used as +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + # When a child rewrites the AOF file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. aof-rewrite-incremental-fsync yes +# When redis saves RDB file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +rdb-save-incremental-fsync yes + +# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good +# idea to start with the default settings and only change them after investigating +# how to improve the performances and how the keys LFU change over time, which +# is possible to inspect via the OBJECT FREQ command. +# +# There are two tunable parameters in the Redis LFU implementation: the +# counter logarithm factor and the counter decay time. It is important to +# understand what the two parameters mean before changing them. +# +# The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis +# uses a probabilistic increment with logarithmic behavior. Given the value +# of the old counter, when a key is accessed, the counter is incremented in +# this way: +# +# 1. A random number R between 0 and 1 is extracted. +# 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). +# 3. The counter is incremented only if R < P. +# +# The default lfu-log-factor is 10. This is a table of how the frequency +# counter changes with a different number of accesses with different +# logarithmic factors: +# +# +--------+------------+------------+------------+------------+------------+ +# | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | +# +--------+------------+------------+------------+------------+------------+ +# | 0 | 104 | 255 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 1 | 18 | 49 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 10 | 10 | 18 | 142 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 100 | 8 | 11 | 49 | 143 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# +# NOTE: The above table was obtained by running the following commands: +# +# redis-benchmark -n 1000000 incr foo +# redis-cli object freq foo +# +# NOTE 2: The counter initial value is 5 in order to give new objects a chance +# to accumulate hits. +# +# The counter decay time is the time, in minutes, that must elapse in order +# for the key counter to be divided by two (or decremented if it has a value +# less <= 10). +# +# The default value for the lfu-decay-time is 1. A Special value of 0 means to +# decay the counter every time it happens to be scanned. +# +# lfu-log-factor 10 +# lfu-decay-time 1 + +########################### ACTIVE DEFRAGMENTATION ####################### +# +# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested +# even in production and manually tested by multiple engineers for some +# time. +# +# What is active defragmentation? +# ------------------------------- +# +# Active (online) defragmentation allows a Redis server to compact the +# spaces left between small allocations and deallocations of data in memory, +# thus allowing to reclaim back memory. +# +# Fragmentation is a natural process that happens with every allocator (but +# less so with Jemalloc, fortunately) and certain workloads. Normally a server +# restart is needed in order to lower the fragmentation, or at least to flush +# away all the data and create it again. However thanks to this feature +# implemented by Oran Agra for Redis 4.0 this process can happen at runtime +# in an "hot" way, while the server is running. +# +# Basically when the fragmentation is over a certain level (see the +# configuration options below) Redis will start to create new copies of the +# values in contiguous memory regions by exploiting certain specific Jemalloc +# features (in order to understand if an allocation is causing fragmentation +# and to allocate it in a better place), and at the same time, will release the +# old copies of the data. This process, repeated incrementally for all the keys +# will cause the fragmentation to drop back to normal values. +# +# Important things to understand: +# +# 1. This feature is disabled by default, and only works if you compiled Redis +# to use the copy of Jemalloc we ship with the source code of Redis. +# This is the default with Linux builds. +# +# 2. You never need to enable this feature if you don't have fragmentation +# issues. +# +# 3. Once you experience fragmentation, you can enable this feature when +# needed with the command "CONFIG SET activedefrag yes". +# +# The configuration parameters are able to fine tune the behavior of the +# defragmentation process. If you are not sure about what they mean it is +# a good idea to leave the defaults untouched. + +# Enabled active defragmentation +# activedefrag yes + +# Minimum amount of fragmentation waste to start active defrag +# active-defrag-ignore-bytes 100mb + +# Minimum percentage of fragmentation to start active defrag +# active-defrag-threshold-lower 10 + +# Maximum percentage of fragmentation at which we use maximum effort +# active-defrag-threshold-upper 100 + +# Minimal effort for defrag in CPU percentage +# active-defrag-cycle-min 5 + +# Maximal effort for defrag in CPU percentage +# active-defrag-cycle-max 75 + +# Maximum number of set/hash/zset/list fields that will be processed from +# the main dictionary scan +# active-defrag-max-scan-fields 1000 diff --git a/server/configs/server.conf.sample b/server/configs/server.conf.sample new file mode 100644 index 0000000..979ceb1 --- /dev/null +++ b/server/configs/server.conf.sample @@ -0,0 +1,5 @@ +[Save_Directories] +# By default all datas are saved in $D4_HOME/data/ +use_default_save_directory = yes +save_directory = None + diff --git a/server/configs/update_conf.py b/server/configs/update_conf.py new file mode 100755 index 0000000..9ee2c3a --- /dev/null +++ b/server/configs/update_conf.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import os +import argparse +import configparser + +def print_message(message_to_print, verbose): + if verbose: + print(message_to_print) + +if __name__ == "__main__": + + # parse parameters + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose',help='Display Info Messages', type=int, default=1, choices=[0, 1]) + parser.add_argument('-b', '--backup',help='Create Config Backup', type=int, default=1, choices=[0, 1]) + args = parser.parse_args() + if args.verbose == 1: + verbose = True + else: + verbose = False + if args.backup == 1: + backup = True + else: + backup = False + + config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') + config_file_sample = os.path.join(os.environ['D4_HOME'], 'configs/server.conf.sample') + config_file_backup = os.path.join(os.environ['D4_HOME'], 'configs/server.conf.backup') + + # Check if confile file exist + if not os.path.isfile(config_file_server): + # create config file + with open(config_file_server, 'w') as configfile: + with open(config_file_sample, 'r') as config_file_sample: + configfile.write(config_file_sample.read()) + print_message('Config File Created', verbose) + else: + config_server = configparser.ConfigParser() + config_server.read(config_file_server) + config_sections = config_server.sections() + + config_sample = configparser.ConfigParser() + config_sample.read(config_file_sample) + sample_sections = config_sample.sections() + + mew_content_added = False + for section in sample_sections: + new_key_added = False + if section not in config_sections: + # add new section + config_server.add_section(section) + mew_content_added = True + for key in config_sample[section]: + if key not in config_server[section]: + # add new section key + config_server.set(section, key, config_sample[section][key]) + if not new_key_added: + print_message('[{}]'.format(section), verbose) + new_key_added = True + mew_content_added = True + print_message(' {} = {}'.format(key, config_sample[section][key]), verbose) + + # new keys have been added to config file + if mew_content_added: + # backup config file + if backup: + with open(config_file_backup, 'w') as configfile: + with open(config_file_server, 'r') as configfile_origin: + configfile.write(configfile_origin.read()) + print_message('New Backup Created', verbose) + # create new config file + with open(config_file_server, 'w') as configfile: + config_server.write(configfile) + print_message('Config file updated', verbose) + else: + print_message('Nothing to update', verbose) diff --git a/server/gen_cert/gen_root.sh b/server/gen_cert/gen_root.sh index 218a7a7..1f6e582 100755 --- a/server/gen_cert/gen_root.sh +++ b/server/gen_cert/gen_root.sh @@ -2,4 +2,4 @@ # Create Root key openssl genrsa -out rootCA.key 4096 # Create and Sign the Root CA Certificate -openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt +openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt -config san.cnf diff --git a/server/install_server.sh b/server/install_server.sh index 292b08b..27a0799 100755 --- a/server/install_server.sh +++ b/server/install_server.sh @@ -3,7 +3,7 @@ set -e set -x -sudo apt-get install python3-pip virtualenv screen -y +sudo apt-get install python3-pip virtualenv screen whois unzip libffi-dev gcc -y if [ -z "$VIRTUAL_ENV" ]; then virtualenv -p python3 D4ENV diff --git a/server/server.py b/server/server.py index d476935..fd33764 100755 --- a/server/server.py +++ b/server/server.py @@ -23,7 +23,8 @@ from twisted.protocols.policies import TimeoutMixin hmac_reset = bytearray(32) hmac_key = b'private key to change' -accepted_type = [1, 4] +accepted_type = [1, 2, 4, 8, 254] +accepted_extended_type = ['ja3-jl'] timeout_time = 30 @@ -60,40 +61,86 @@ except redis.exceptions.ConnectionError: print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_metadata, port_redis_metadata)) sys.exit(1) +# set hmac default key +redis_server_metadata.set('server:hmac_default_key', hmac_key) + # init redis_server_metadata redis_server_metadata.delete('server:accepted_type') for type in accepted_type: redis_server_metadata.sadd('server:accepted_type', type) +redis_server_metadata.delete('server:accepted_extended_type') +for type in accepted_extended_type: + redis_server_metadata.sadd('server:accepted_extended_type', type) -class Echo(Protocol, TimeoutMixin): +dict_all_connection = {} + +class D4_Server(Protocol, TimeoutMixin): def __init__(self): self.buffer = b'' self.setTimeout(timeout_time) self.session_uuid = str(uuid.uuid4()) self.data_saved = False + self.update_stream_type = True + self.first_connection = True + self.ip = None + self.source_port = None self.stream_max_size = None + self.hmac_key = None + #self.version = None + self.type = None + self.uuid = None logger.debug('New session: session_uuid={}'.format(self.session_uuid)) + dict_all_connection[self.session_uuid] = self def dataReceived(self, data): - self.resetTimeout() - ip, source_port = self.transport.client - # check blacklisted_ip - if redis_server_metadata.sismember('blacklist_ip', ip): - self.transport.abortConnection() - logger.warning('Blacklisted IP={}, connection closed'.format(ip)) + # check and kick sensor by uuid + for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'): + client_uuid = client_uuid.decode() + for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)): + session_uuid = session_uuid.decode() + logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid)) + redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string') + redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30) + dict_all_connection[session_uuid].transport.abortConnection() + redis_server_stream.srem('server:sensor_to_kick', client_uuid) - self.process_header(data, ip, source_port) + self.resetTimeout() + if self.first_connection or self.ip is None: + client_info = self.transport.client + self.ip = self.extract_ip(client_info[0]) + self.source_port = client_info[1] + logger.debug('New connection, ip={}, port={} session_uuid={}'.format(self.ip, self.source_port, self.session_uuid)) + # check blacklisted_ip + if redis_server_metadata.sismember('blacklist_ip', self.ip): + self.transport.abortConnection() + logger.warning('Blacklisted IP={}, connection closed'.format(self.ip)) + else: + # process data + self.process_header(data, self.ip, self.source_port) def timeoutConnection(self): - self.resetTimeout() - self.buffer = b'' - logger.debug('buffer timeout, session_uuid={}'.format(self.session_uuid)) + if self.uuid is None: + # # TODO: ban auto + logger.warning('Timeout, no D4 header send, session_uuid={}, connection closed'.format(self.session_uuid)) + self.transport.abortConnection() + else: + self.resetTimeout() + self.buffer = b'' + logger.debug('buffer timeout, session_uuid={}'.format(self.session_uuid)) + + def connectionMade(self): + self.transport.setTcpKeepAlive(1) def connectionLost(self, reason): redis_server_stream.sadd('ended_session', self.session_uuid) self.setTimeout(None) + redis_server_stream.srem('active_connection:{}'.format(self.type), '{}:{}'.format(self.ip, self.uuid)) + redis_server_stream.srem('active_connection', '{}'.format(self.uuid)) + if self.uuid: + redis_server_stream.srem('map:active_connection-uuid-session_uuid:{}'.format(self.uuid), self.session_uuid) logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid)) + dict_all_connection.pop(self.session_uuid) def unpack_header(self, data): data_header = {} @@ -104,25 +151,18 @@ class Echo(Protocol, TimeoutMixin): data_header['timestamp'] = struct.unpack('Q', data[18:26])[0] data_header['hmac_header'] = data[26:58] data_header['size'] = struct.unpack('I', data[58:62])[0] + return data_header - # uuid blacklist - if redis_server_metadata.sismember('blacklist_uuid', data_header['uuid_header']): - self.transport.abortConnection() - logger.warning('Blacklisted UUID={}, connection closed'.format(data_header['uuid_header'])) - - # check default size limit - if data_header['size'] > data_default_size_limit: - self.transport.abortConnection() - logger.warning('Incorrect header data size: the server received more data than expected by default, expected={}, received={} , uuid={}, session_uuid={}'.format(data_default_size_limit, data_header['size'] ,data_header['uuid_header'], self.session_uuid)) - - # Worker: Incorrect type - if redis_server_stream.sismember('Error:IncorrectType:{}'.format(data_header['type']), self.session_uuid): - self.transport.abortConnection() - redis_server_stream.delete(stream_name) - redis_server_stream.srem('Error:IncorrectType:{}'.format(data_header['type']), self.session_uuid) - logger.warning('Incorrect type={} detected by worker, uuid={}, session_uuid={}'.format(data_header['type'] ,data_header['uuid_header'], self.session_uuid)) - - return data_header + def extract_ip(self, ip_string): + #remove interface + ip_string = ip_string.split('%')[0] + # IPv4 + #extract ipv4 + if '.' in ip_string: + return ip_string.split(':')[-1] + # IPv6 + else: + return ip_string def is_valid_uuid_v4(self, header_uuid): try: @@ -143,14 +183,103 @@ class Echo(Protocol, TimeoutMixin): logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid)) return False + def check_connection_validity(self, data_header): + # blacklist ip by uuid + if redis_server_metadata.sismember('blacklist_ip_by_uuid', data_header['uuid_header']): + redis_server_metadata.sadd('blacklist_ip', self.ip) + self.transport.abortConnection() + logger.warning('Blacklisted IP by UUID={}, connection closed'.format(data_header['uuid_header'])) + return False + + # uuid blacklist + if redis_server_metadata.sismember('blacklist_uuid', data_header['uuid_header']): + logger.warning('Blacklisted UUID={}, connection closed'.format(data_header['uuid_header'])) + self.transport.abortConnection() + return False + + # check temp blacklist + if redis_server_stream.exists('temp_blacklist_uuid:{}'.format(data_header['uuid_header'])): + logger.warning('Temporarily Blacklisted UUID={}, connection closed'.format(data_header['uuid_header'])) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted') + self.transport.abortConnection() + return False + + # check default size limit + if data_header['size'] > data_default_size_limit: + self.transport.abortConnection() + logger.warning('Incorrect header data size: the server received more data than expected by default, expected={}, received={} , uuid={}, session_uuid={}'.format(data_default_size_limit, data_header['size'] ,data_header['uuid_header'], self.session_uuid)) + return False + + # Worker: Incorrect type + if redis_server_stream.sismember('Error:IncorrectType', self.session_uuid): + self.transport.abortConnection() + redis_server_stream.delete('stream:{}:{}'.format(data_header['type'], self.session_uuid)) + redis_server_stream.srem('Error:IncorrectType', self.session_uuid) + logger.warning('Incorrect type={} detected by worker, uuid={}, session_uuid={}'.format(data_header['type'] ,data_header['uuid_header'], self.session_uuid)) + return False + + return True + def process_header(self, data, ip, source_port): if not self.buffer: data_header = self.unpack_header(data) if data_header: + if not self.check_connection_validity(data_header): + return 1 if self.is_valid_header(data_header['uuid_header'], data_header['type']): + + # auto kill connection # TODO: map type + if self.first_connection: + self.first_connection = False + if redis_server_stream.sismember('active_connection:{}'.format(data_header['type']), '{}:{}'.format(ip, data_header['uuid_header'])): + # same IP-type for an UUID + logger.warning('is using the same UUID for one type, ip={} uuid={} type={} session_uuid={}'.format(ip, data_header['uuid_header'], data_header['type'], self.session_uuid)) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is using the same UUID for one type={}'.format(data_header['type'])) + self.transport.abortConnection() + return 1 + else: + #self.version = None + # check if type change + if self.data_saved: + # type change detected + if self.type != data_header['type']: + # Meta types + if self.type == 2 and data_header['type'] == 254: + self.update_stream_type = True + # Type Error + else: + logger.warning('Unexpected type change, type={} new type={}, ip={} uuid={} session_uuid={}'.format(ip, data_header['uuid_header'], data_header['type'], self.session_uuid)) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: Unexpected type change type={}, new type={}'.format(self.type, data_header['type'])) + self.transport.abortConnection() + return 1 + # type 254, check if previous type 2 saved + elif data_header['type'] == 254: + logger.warning('a type 2 packet must be sent, ip={} uuid={} type={} session_uuid={}'.format(ip, data_header['uuid_header'], data_header['type'], self.session_uuid)) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: a type 2 packet must be sent, type={}'.format(data_header['type'])) + self.transport.abortConnection() + return 1 + self.type = data_header['type'] + self.uuid = data_header['uuid_header'] + #active Connection + redis_server_stream.sadd('active_connection:{}'.format(self.type), '{}:{}'.format(ip, self.uuid)) + redis_server_stream.sadd('active_connection', '{}'.format(self.uuid)) + # map session_uuid/uuid + redis_server_stream.sadd('map:active_connection-uuid-session_uuid:{}'.format(self.uuid), self.session_uuid) + + # check if the uuid is the same + if self.uuid != data_header['uuid_header']: + logger.warning('The uuid change during the connection, ip={} uuid={} type={} session_uuid={} new_uuid={}'.format(ip, self.uuid, data_header['type'], self.session_uuid, data_header['uuid_header'])) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: The uuid change, new_uuid={}'.format(data_header['uuid_header'])) + self.transport.abortConnection() + return 1 + ## TODO: ban ? + # check data size if data_header['size'] == (len(data) - header_size): - self.process_d4_data(data, data_header, ip) + res = self.process_d4_data(data, data_header, ip) + # Error detected, kill connection + if res == 1: + return 1 # multiple d4 headers elif data_header['size'] < (len(data) - header_size): next_data = data[data_header['size'] + header_size:] @@ -159,7 +288,10 @@ class Echo(Protocol, TimeoutMixin): #print(data) #print() #print(next_data) - self.process_d4_data(data, data_header, ip) + res = self.process_d4_data(data, data_header, ip) + # Error detected, kill connection + if res == 1: + return 1 # process next d4 header self.process_header(next_data, ip, source_port) # data_header['size'] > (len(data) - header_size) @@ -210,7 +342,12 @@ class Echo(Protocol, TimeoutMixin): self.buffer = b'' # set hmac_header to 0 data = data.replace(data_header['hmac_header'], hmac_reset, 1) - HMAC = hmac.new(hmac_key, msg=data, digestmod='sha256') + if self.hmac_key is None: + self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(data_header['uuid_header']), 'hmac_key') + if self.hmac_key is None: + self.hmac_key = redis_server_metadata.get('server:hmac_default_key') + + HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256') data_header['hmac_header'] = data_header['hmac_header'].hex() ### Debug ### @@ -234,6 +371,8 @@ class Echo(Protocol, TimeoutMixin): date = datetime.datetime.now().strftime("%Y%m%d") if redis_server_stream.xlen('stream:{}:{}'.format(data_header['type'], self.session_uuid)) < self.stream_max_size: + # Clean Error Message + redis_server_metadata.hdel('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error') redis_server_stream.xadd('stream:{}:{}'.format(data_header['type'], self.session_uuid), {'message': data[header_size:], 'uuid': data_header['uuid_header'], 'timestamp': data_header['timestamp'], 'version': data_header['version']}) @@ -244,24 +383,44 @@ class Echo(Protocol, TimeoutMixin): redis_server_metadata.zincrby('daily_ip:{}'.format(date), 1, ip) redis_server_metadata.zincrby('daily_type:{}'.format(date), 1, data_header['type']) redis_server_metadata.zincrby('stat_type_uuid:{}:{}'.format(date, data_header['type']), 1, data_header['uuid_header']) + redis_server_metadata.zincrby('stat_uuid_type:{}:{}'.format(date, data_header['uuid_header']), 1, data_header['type']) # if not redis_server_metadata.hexists('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen'): redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', data_header['timestamp']) redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', data_header['timestamp']) + redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', data_header['timestamp']) if not self.data_saved: + #UUID IP: ## TODO: use d4 timestamp ? + redis_server_metadata.lpush('list_uuid_ip:{}'.format(data_header['uuid_header']), '{}-{}'.format(ip, datetime.datetime.now().strftime("%Y%m%d%H%M%S"))) + redis_server_metadata.ltrim('list_uuid_ip:{}'.format(data_header['uuid_header']), 0, 15) + + self.data_saved = True + if self.update_stream_type: redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode()) redis_server_stream.hset('map-type:session_uuid-uuid:{}'.format(data_header['type']), self.session_uuid, data_header['uuid_header']) - self.data_saved = True + redis_server_metadata.sadd('all_types_by_uuid:{}'.format(data_header['uuid_header']), data_header['type']) + + if not redis_server_metadata.hexists('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen'): + redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', data_header['timestamp']) + self.update_stream_type = False + return 0 else: logger.warning("stream exceed max entries limit, uuid={}, session_uuid={}, type={}".format(data_header['uuid_header'], self.session_uuid, data_header['type'])) + ## TODO: FIXME + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: stream exceed max entries limit') + self.transport.abortConnection() + return 1 else: print('hmac do not match') print(data) logger.debug("HMAC don't match, uuid={}, session_uuid={}".format(data_header['uuid_header'], self.session_uuid)) - + ## TODO: FIXME + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: HMAC don\'t match') + self.transport.abortConnection() + return 1 def main(reactor): @@ -273,8 +432,9 @@ def main(reactor): print(e) sys.exit(1) certificate = ssl.PrivateCertificate.loadPEM(certData) - factory = protocol.Factory.forProtocol(Echo) - reactor.listenSSL(4443, factory, certificate.options()) + factory = protocol.Factory.forProtocol(D4_Server) + # use interface to support both IPv4 and IPv6 + reactor.listenSSL(4443, factory, certificate.options(), interface='::') return defer.Deferred() @@ -283,6 +443,9 @@ if __name__ == "__main__": parser.add_argument('-v', '--verbose',help='dddd' , type=int, default=30) args = parser.parse_args() + if not redis_server_metadata.exists('first_date'): + redis_server_metadata.set('first_date', datetime.datetime.now().strftime("%Y%m%d")) + logs_dir = 'logs' if not os.path.isdir(logs_dir): os.makedirs(logs_dir) @@ -298,4 +461,5 @@ if __name__ == "__main__": logger.setLevel(args.verbose) logger.info('Launching Server ...') + task.react(main) diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index f000f4d..30a20b5 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -2,11 +2,17 @@ # -*-coding:UTF-8 -* import os +import re import sys +import uuid import time +import json import redis import flask import datetime +import ipaddress + +import subprocess from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for @@ -14,6 +20,22 @@ baseUrl = '' if baseUrl != '': baseUrl = '/'+baseUrl +host_redis_stream = "localhost" +port_redis_stream = 6379 + +default_max_entries_by_stream = 10000 +analyzer_list_max_default_size = 10000 + +default_analyzer_max_line_len = 3000 + +json_type_description_path = os.path.join(os.environ['D4_HOME'], 'web/static/json/type.json') + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0, + decode_responses=True) + host_redis_metadata = "localhost" port_redis_metadata= 6380 @@ -23,13 +45,89 @@ redis_server_metadata = redis.StrictRedis( db=0, decode_responses=True) +redis_server_analyzer = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=2, + decode_responses=True) + +with open(json_type_description_path, 'r') as f: + json_type = json.loads(f.read()) +json_type_description = {} +for type_info in json_type: + json_type_description[type_info['type']] = type_info + app = Flask(__name__, static_url_path=baseUrl+'/static/') app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024 +# ========== FUNCTIONS ============ +def is_valid_uuid_v4(header_uuid): + try: + header_uuid=header_uuid.replace('-', '') + uuid_test = uuid.UUID(hex=header_uuid, version=4) + return uuid_test.hex == header_uuid + except: + return False + +def is_valid_ip(ip): + try: + ipaddress.ip_address(ip) + return True + except ValueError: + return False + +def is_valid_network(ip_network): + try: + ipaddress.ip_network(ip_network) + return True + except ValueError: + return False + +# server_management input handler +def get_server_management_input_handler_value(value): + if value is not None: + if value !="0": + try: + value=int(value) + except: + value=0 + else: + value=0 + return value + +def get_json_type_description(): + return json_type_description + +def get_whois_ouput(ip): + if is_valid_ip(ip): + process = subprocess.run(["whois", ip], stdout=subprocess.PIPE) + return re.sub(r"#.*\n?", '', process.stdout.decode()).lstrip('\n').rstrip('\n') + else: + return '' + +def get_substract_date_range(num_day, date_from=None): + if date_from is None: + date_from = datetime.datetime.now() + else: + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + + l_date = [] + for i in range(num_day): + date = date_from - datetime.timedelta(days=i) + l_date.append( date.strftime('%Y%m%d') ) + return list(reversed(l_date)) + +# ========== ERRORS ============ + +@app.errorhandler(404) +def page_not_found(e): + return render_template('404.html'), 404 + # ========== ROUTES ============ @app.route('/') def index(): - return render_template("index.html") + date = datetime.datetime.now().strftime("%Y/%m/%d") + return render_template("index.html", date=date) @app.route('/_json_daily_uuid_stats') def _json_daily_uuid_stats(): @@ -42,5 +140,632 @@ def _json_daily_uuid_stats(): return jsonify(data_daily_uuid) +@app.route('/_json_daily_type_stats') +def _json_daily_type_stats(): + date = datetime.datetime.now().strftime("%Y%m%d") + daily_uuid = redis_server_metadata.zrange('daily_type:{}'.format(date), 0, -1, withscores=True) + json_type_description = get_json_type_description() + + data_daily_uuid = [] + for result in daily_uuid: + try: + type_description = json_type_description[int(result[0])]['description'] + except: + type_description = 'Please update your web server' + data_daily_uuid.append({"key": '{}: {}'.format(result[0], type_description), "value": int(result[1])}) + + return jsonify(data_daily_uuid) + +@app.route('/sensors_status') +def sensors_status(): + active_connection_filter = request.args.get('active_connection_filter') + if active_connection_filter is None: + active_connection_filter = False + else: + if active_connection_filter=='True': + active_connection_filter = True + else: + active_connection_filter = False + + date = datetime.datetime.now().strftime("%Y%m%d") + + if not active_connection_filter: + daily_uuid = redis_server_metadata.zrange('daily_uuid:{}'.format(date), 0, -1) + else: + daily_uuid = redis_server_stream.smembers('active_connection') + + status_daily_uuid = [] + for result in daily_uuid: + first_seen = redis_server_metadata.hget('metadata_uuid:{}'.format(result), 'first_seen') + first_seen_gmt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(first_seen))) + last_seen = redis_server_metadata.hget('metadata_uuid:{}'.format(result), 'last_seen') + last_seen_gmt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(last_seen))) + if redis_server_metadata.sismember('blacklist_ip_by_uuid', result): + Error = "All IP using this UUID are Blacklisted" + elif redis_server_metadata.sismember('blacklist_uuid', result): + Error = "Blacklisted UUID" + else: + Error = redis_server_metadata.hget('metadata_uuid:{}'.format(result), 'Error') + if redis_server_stream.sismember('active_connection', result): + active_connection = True + else: + active_connection = False + + if first_seen is not None and last_seen is not None: + status_daily_uuid.append({"uuid": result,"first_seen": first_seen, "last_seen": last_seen, + "active_connection": active_connection, + "first_seen_gmt": first_seen_gmt, "last_seen_gmt": last_seen_gmt, "Error": Error}) + + return render_template("sensors_status.html", status_daily_uuid=status_daily_uuid, + active_connection_filter=active_connection_filter) + +@app.route('/show_active_uuid') +def show_active_uuid(): + #swap switch value + active_connection_filter = request.args.get('show_active_connection') + if active_connection_filter is None: + active_connection_filter = True + else: + if active_connection_filter=='True': + active_connection_filter = False + else: + active_connection_filter = True + + return redirect(url_for('sensors_status', active_connection_filter=active_connection_filter)) + +@app.route('/server_management') +def server_management(): + blacklisted_ip = request.args.get('blacklisted_ip') + unblacklisted_ip = request.args.get('unblacklisted_ip') + blacklisted_uuid = request.args.get('blacklisted_uuid') + unblacklisted_uuid = request.args.get('unblacklisted_uuid') + + blacklisted_ip = get_server_management_input_handler_value(blacklisted_ip) + unblacklisted_ip = get_server_management_input_handler_value(unblacklisted_ip) + blacklisted_uuid = get_server_management_input_handler_value(blacklisted_uuid) + unblacklisted_uuid = get_server_management_input_handler_value(unblacklisted_uuid) + + json_type_description = get_json_type_description() + + list_accepted_types = [] + list_analyzer_types = [] + for type in redis_server_metadata.smembers('server:accepted_type'): + try: + description = json_type_description[int(type)]['description'] + except: + description = 'Please update your web server' + + list_analyzer_uuid = [] + for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)): + size_limit = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if size_limit is None: + size_limit = analyzer_list_max_default_size + last_updated = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'last_updated') + if last_updated is None: + last_updated = 'Never' + else: + last_updated = datetime.datetime.fromtimestamp(float(last_updated)).strftime('%Y-%m-%d %H:%M:%S') + description_analyzer = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'description') + if description_analyzer is None: + description_analyzer = '' + len_queue = redis_server_analyzer.llen('analyzer:{}:{}'.format(type, analyzer_uuid)) + if len_queue is None: + len_queue = 0 + list_analyzer_uuid.append({'uuid': analyzer_uuid, 'description': description_analyzer, 'size_limit': size_limit,'last_updated': last_updated, 'length': len_queue}) + + list_accepted_types.append({"id": int(type), "description": description, 'list_analyzer_uuid': list_analyzer_uuid}) + + list_accepted_extended_types = [] + for extended_type in redis_server_metadata.smembers('server:accepted_extended_type'): + + list_analyzer_uuid = [] + for analyzer_uuid in redis_server_metadata.smembers('analyzer:254:{}'.format(extended_type)): + size_limit = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if size_limit is None: + size_limit = analyzer_list_max_default_size + last_updated = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'last_updated') + if last_updated is None: + last_updated = 'Never' + else: + last_updated = datetime.datetime.fromtimestamp(float(last_updated)).strftime('%Y-%m-%d %H:%M:%S') + description_analyzer = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'description') + if description_analyzer is None: + description_analyzer = '' + len_queue = redis_server_analyzer.llen('analyzer:{}:{}'.format(extended_type, analyzer_uuid)) + if len_queue is None: + len_queue = 0 + list_analyzer_uuid.append({'uuid': analyzer_uuid, 'description': description_analyzer, 'size_limit': size_limit,'last_updated': last_updated, 'length': len_queue}) + + list_accepted_extended_types.append({"name": extended_type, 'list_analyzer_uuid': list_analyzer_uuid}) + + return render_template("server_management.html", list_accepted_types=list_accepted_types, list_accepted_extended_types=list_accepted_extended_types, + default_analyzer_max_line_len=default_analyzer_max_line_len, + blacklisted_ip=blacklisted_ip, unblacklisted_ip=unblacklisted_ip, + blacklisted_uuid=blacklisted_uuid, unblacklisted_uuid=unblacklisted_uuid) + +@app.route('/uuid_management') +def uuid_management(): + uuid_sensor = request.args.get('uuid') + if is_valid_uuid_v4(uuid_sensor): + + first_seen = redis_server_metadata.hget('metadata_uuid:{}'.format(uuid_sensor), 'first_seen') + first_seen_gmt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(first_seen))) + last_seen = redis_server_metadata.hget('metadata_uuid:{}'.format(uuid_sensor), 'last_seen') + last_seen_gmt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(last_seen))) + Error = redis_server_metadata.hget('metadata_uuid:{}'.format(uuid_sensor), 'Error') + if redis_server_stream.exists('temp_blacklist_uuid:{}'.format(uuid_sensor)): + temp_blacklist_uuid = True + else: + temp_blacklist_uuid = False + if redis_server_metadata.sismember('blacklist_uuid', uuid_sensor): + blacklisted_uuid = True + Error = "Blacklisted UUID" + else: + blacklisted_uuid = False + if redis_server_metadata.sismember('blacklist_ip_by_uuid', uuid_sensor): + blacklisted_ip_by_uuid = True + Error = "All IP using this UUID are Blacklisted" + else: + blacklisted_ip_by_uuid = False + data_uuid= {"first_seen": first_seen, "last_seen": last_seen, + "temp_blacklist_uuid": temp_blacklist_uuid, + "blacklisted_uuid": blacklisted_uuid, "blacklisted_ip_by_uuid": blacklisted_ip_by_uuid, + "first_seen_gmt": first_seen_gmt, "last_seen_gmt": last_seen_gmt, "Error": Error} + + if redis_server_stream.sismember('active_connection', uuid_sensor): + active_connection = True + else: + active_connection = False + + max_uuid_stream = redis_server_metadata.hget('stream_max_size_by_uuid', uuid_sensor) + if max_uuid_stream is not None: + max_uuid_stream = int(max_uuid_stream) + else: + max_uuid_stream = default_max_entries_by_stream + + uuid_key = redis_server_metadata.hget('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key') + if uuid_key is None: + uuid_key = redis_server_metadata.get('server:hmac_default_key') + + uuid_all_type_list = [] + uuid_all_type = redis_server_metadata.smembers('all_types_by_uuid:{}'.format(uuid_sensor)) + for type in uuid_all_type: + type_first_seen = redis_server_metadata.hget('metadata_type_by_uuid:{}:{}'.format(uuid_sensor, type), 'first_seen') + type_last_seen = redis_server_metadata.hget('metadata_type_by_uuid:{}:{}'.format(uuid_sensor, type), 'last_seen') + if type_first_seen: + type_first_seen = datetime.datetime.fromtimestamp(float(type_first_seen)).strftime('%Y-%m-%d %H:%M:%S') + if type_last_seen: + type_last_seen = datetime.datetime.fromtimestamp(float(type_last_seen)).strftime('%Y-%m-%d %H:%M:%S') + uuid_all_type_list.append({'type': type, 'first_seen':type_first_seen, 'last_seen': type_last_seen}) + + list_ip = redis_server_metadata.lrange('list_uuid_ip:{}'.format(uuid_sensor), 0, -1) + all_ip = [] + for elem in list_ip: + ip, d_time = elem.split('-') + all_ip.append({'ip': ip,'datetime': '{}/{}/{} - {}:{}.{}'.format(d_time[0:4], d_time[5:6], d_time[6:8], d_time[8:10], d_time[10:12], d_time[12:14])}) + + return render_template("uuid_management.html", uuid_sensor=uuid_sensor, active_connection=active_connection, + uuid_key=uuid_key, data_uuid=data_uuid, uuid_all_type=uuid_all_type_list, + max_uuid_stream=max_uuid_stream, all_ip=all_ip) + else: + return 'Invalid uuid' + +@app.route('/blacklisted_ip') +def blacklisted_ip(): + blacklisted_ip = request.args.get('blacklisted_ip') + unblacklisted_ip = request.args.get('unblacklisted_ip') + try: + page = int(request.args.get('page')) + except: + page = 1 + if page <= 0: + page = 1 + nb_page_max = redis_server_metadata.scard('blacklist_ip')/(1000*2) + if isinstance(nb_page_max, float): + nb_page_max = int(nb_page_max)+1 + if page > nb_page_max: + page = nb_page_max + start = 1000*(page -1) + stop = 1000*page + + list_blacklisted_ip = list(redis_server_metadata.smembers('blacklist_ip')) + list_blacklisted_ip_1 = list_blacklisted_ip[start:stop] + list_blacklisted_ip_2 = list_blacklisted_ip[stop:stop+1000] + return render_template("blacklisted_ip.html", list_blacklisted_ip_1=list_blacklisted_ip_1, list_blacklisted_ip_2=list_blacklisted_ip_2, + page=page, nb_page_max=nb_page_max, + unblacklisted_ip=unblacklisted_ip, blacklisted_ip=blacklisted_ip) + +@app.route('/blacklisted_uuid') +def blacklisted_uuid(): + blacklisted_uuid = request.args.get('blacklisted_uuid') + unblacklisted_uuid = request.args.get('unblacklisted_uuid') + try: + page = int(request.args.get('page')) + except: + page = 1 + if page <= 0: + page = 1 + nb_page_max = redis_server_metadata.scard('blacklist_uuid')/(1000*2) + if isinstance(nb_page_max, float): + nb_page_max = int(nb_page_max)+1 + if page > nb_page_max: + page = nb_page_max + start = 1000*(page -1) + stop = 1000*page + + list_blacklisted_uuid = list(redis_server_metadata.smembers('blacklist_uuid')) + list_blacklisted_uuid_1 = list_blacklisted_uuid[start:stop] + list_blacklisted_uuid_2 = list_blacklisted_uuid[stop:stop+1000] + return render_template("blacklisted_uuid.html", list_blacklisted_uuid_1=list_blacklisted_uuid_1, list_blacklisted_uuid_2=list_blacklisted_uuid_2, + page=page, nb_page_max=nb_page_max, + unblacklisted_uuid=unblacklisted_uuid, blacklisted_uuid=blacklisted_uuid) + + +@app.route('/uuid_change_stream_max_size') +def uuid_change_stream_max_size(): + uuid_sensor = request.args.get('uuid') + user = request.args.get('redirect') + max_uuid_stream = request.args.get('max_uuid_stream') + if is_valid_uuid_v4(uuid_sensor): + try: + max_uuid_stream = int(max_uuid_stream) + if max_uuid_stream < 0: + return 'stream max size, Invalid Integer' + except: + return 'stream max size, Invalid Integer' + redis_server_metadata.hset('stream_max_size_by_uuid', uuid_sensor, max_uuid_stream) + if user: + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + else: + return 'Invalid uuid' + +# # TODO: check analyser uuid dont exist +@app.route('/add_new_analyzer') +def add_new_analyzer(): + type = request.args.get('type') + user = request.args.get('redirect') + metatype_name = request.args.get('metatype_name') + analyzer_description = request.args.get('analyzer_description') + analyzer_uuid = request.args.get('analyzer_uuid') + if is_valid_uuid_v4(analyzer_uuid): + try: + type = int(type) + if type < 0: + return 'type, Invalid Integer' + except: + return 'type, Invalid Integer' + if type == 254: + # # TODO: check metatype_name + redis_server_metadata.sadd('analyzer:{}:{}'.format(type, metatype_name), analyzer_uuid) + else: + redis_server_metadata.sadd('analyzer:{}'.format(type), analyzer_uuid) + if redis_server_metadata.exists('analyzer:{}:{}'.format(type, metatype_name)) or redis_server_metadata.exists('analyzer:{}'.format(type)): + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'description', analyzer_description) + if user: + return redirect(url_for('server_management')) + else: + return 'Invalid uuid' + +@app.route('/empty_analyzer_queue') +def empty_analyzer_queue(): + analyzer_uuid = request.args.get('analyzer_uuid') + type = request.args.get('type') + metatype_name = request.args.get('metatype_name') + user = request.args.get('redirect') + if is_valid_uuid_v4(analyzer_uuid): + try: + type = int(type) + if type < 0: + return 'type, Invalid Integer' + except: + return 'type, Invalid Integer' + if type == 254: + redis_server_analyzer.delete('analyzer:{}:{}'.format(metatype_name, analyzer_uuid)) + else: + redis_server_analyzer.delete('analyzer:{}:{}'.format(type, analyzer_uuid)) + if user: + return redirect(url_for('server_management')) + else: + return 'Invalid uuid' + +@app.route('/remove_analyzer') +def remove_analyzer(): + analyzer_uuid = request.args.get('analyzer_uuid') + type = request.args.get('type') + metatype_name = request.args.get('metatype_name') + user = request.args.get('redirect') + if is_valid_uuid_v4(analyzer_uuid): + try: + type = int(type) + if type < 0: + return 'type, Invalid Integer' + except: + return 'type, Invalid Integer' + if type == 254: + redis_server_metadata.srem('analyzer:{}:{}'.format(type, metatype_name), analyzer_uuid) + redis_server_analyzer.delete('analyzer:{}:{}'.format(metatype_name, analyzer_uuid)) + else: + redis_server_metadata.srem('analyzer:{}'.format(type), analyzer_uuid) + redis_server_analyzer.delete('analyzer:{}:{}'.format(type, analyzer_uuid)) + redis_server_metadata.delete('analyzer:{}'.format(analyzer_uuid)) + if user: + return redirect(url_for('server_management')) + else: + return 'Invalid uuid' + +@app.route('/analyzer_change_max_size') +def analyzer_change_max_size(): + analyzer_uuid = request.args.get('analyzer_uuid') + user = request.args.get('redirect') + max_size_analyzer = request.args.get('max_size_analyzer') + if is_valid_uuid_v4(analyzer_uuid): + try: + max_size_analyzer = int(max_size_analyzer) + if max_size_analyzer < 0: + return 'analyzer max size, Invalid Integer' + except: + return 'analyzer max size, Invalid Integer' + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'max_size', max_size_analyzer) + if user: + return redirect(url_for('server_management')) + else: + return 'Invalid uuid' + +@app.route('/kick_uuid') +def kick_uuid(): + uuid_sensor = request.args.get('uuid') + if is_valid_uuid_v4(uuid_sensor): + redis_server_stream.sadd('server:sensor_to_kick', uuid_sensor) + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + else: + return 'Invalid uuid' + +@app.route('/blacklist_uuid') +def blacklist_uuid(): + uuid_sensor = request.args.get('uuid') + user = request.args.get('redirect') + if is_valid_uuid_v4(uuid_sensor): + res = redis_server_metadata.sadd('blacklist_uuid', uuid_sensor) + if user=="0": + if res==0: + return redirect(url_for('server_management', blacklisted_uuid=2)) + else: + return redirect(url_for('server_management', blacklisted_uuid=1)) + elif user=="1": + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + else: + return "404" + else: + if user=="0": + return redirect(url_for('server_management', blacklisted_uuid=0)) + return 'Invalid uuid' + +@app.route('/unblacklist_uuid') +def unblacklist_uuid(): + uuid_sensor = request.args.get('uuid') + user = request.args.get('redirect') + page = request.args.get('page') + if is_valid_uuid_v4(uuid_sensor): + res = redis_server_metadata.srem('blacklist_uuid', uuid_sensor) + if page: + return redirect(url_for('blacklisted_uuid', page=page)) + if user=="0": + if res==0: + return redirect(url_for('server_management', unblacklisted_uuid=2)) + else: + return redirect(url_for('server_management', unblacklisted_uuid=1)) + elif user=="1": + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + else: + return "404" + else: + if user=="0": + return redirect(url_for('server_management', unblacklisted_uuid=0)) + return 'Invalid uuid' + +@app.route('/blacklist_ip') +def blacklist_ip(): + ip = request.args.get('ip') + user = request.args.get('redirect') + + if is_valid_ip(ip): + res = redis_server_metadata.sadd('blacklist_ip', ip) + if user: + if res==0: + return redirect(url_for('server_management', blacklisted_ip=2)) + else: + return redirect(url_for('server_management', blacklisted_ip=1)) + elif is_valid_network(ip): + for addr in ipaddress.ip_network(ip): + res = redis_server_metadata.sadd('blacklist_ip', str(addr)) + if user: + if res==0: + return redirect(url_for('server_management', blacklisted_ip=2)) + else: + return redirect(url_for('server_management', blacklisted_ip=1)) + else: + if user: + return redirect(url_for('server_management', blacklisted_ip=0)) + return 'Invalid ip' + +@app.route('/unblacklist_ip') +def unblacklist_ip(): + ip = request.args.get('ip') + user = request.args.get('redirect') + page = request.args.get('page') + if is_valid_ip(ip): + res = redis_server_metadata.srem('blacklist_ip', ip) + if page: + return redirect(url_for('blacklisted_ip', page=page)) + if user: + if res==0: + return redirect(url_for('server_management', unblacklisted_ip=2)) + else: + return redirect(url_for('server_management', unblacklisted_ip=1)) + elif is_valid_network(ip): + for addr in ipaddress.ip_network(ip): + res = redis_server_metadata.srem('blacklist_ip', str(addr)) + if user: + if res==0: + return redirect(url_for('server_management', unblacklisted_ip=2)) + else: + return redirect(url_for('server_management', unblacklisted_ip=1)) + else: + if user: + return redirect(url_for('server_management', unblacklisted_ip=0)) + return 'Invalid ip' + +@app.route('/blacklist_ip_by_uuid') +def blacklist_ip_by_uuid(): + uuid_sensor = request.args.get('uuid') + user = request.args.get('redirect') + if is_valid_uuid_v4(uuid_sensor): + redis_server_metadata.sadd('blacklist_ip_by_uuid', uuid_sensor) + if user: + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + else: + return 'Invalid uuid' + +@app.route('/unblacklist_ip_by_uuid') +def unblacklist_ip_by_uuid(): + uuid_sensor = request.args.get('uuid') + user = request.args.get('redirect') + if is_valid_uuid_v4(uuid_sensor): + redis_server_metadata.srem('blacklist_ip_by_uuid', uuid_sensor) + if user: + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + else: + return 'Invalid uuid' + +@app.route('/add_accepted_type') +def add_accepted_type(): + type = request.args.get('type') + extended_type_name = request.args.get('extended_type_name') + user = request.args.get('redirect') + json_type_description = get_json_type_description() + try: + type = int(type) + except: + return 'Invalid type' + if json_type_description[int(type)]: + redis_server_metadata.sadd('server:accepted_type', type) + if type == 254: + redis_server_metadata.sadd('server:accepted_extended_type', extended_type_name) + if user: + return redirect(url_for('server_management')) + else: + return 'Invalid type' + +@app.route('/remove_accepted_type') +def remove_accepted_type(): + type = request.args.get('type') + user = request.args.get('redirect') + json_type_description = get_json_type_description() + if json_type_description[int(type)]: + redis_server_metadata.srem('server:accepted_type', type) + if user: + return redirect(url_for('server_management')) + else: + return 'Invalid type' + +@app.route('/remove_accepted_extended_type') +def remove_accepted_extended_type(): + type_name = request.args.get('type_name') + redis_server_metadata.srem('server:accepted_extended_type', type_name) + return redirect(url_for('server_management')) + +# demo function +@app.route('/delete_data') +def delete_data(): + date = datetime.datetime.now().strftime("%Y%m%d") + redis_server_metadata.delete('daily_type:{}'.format(date)) + redis_server_metadata.delete('daily_uuid:{}'.format(date)) + return render_template("index.html") + +# demo function +@app.route('/set_uuid_hmac_key') +def set_uuid_hmac_key(): + uuid_sensor = request.args.get('uuid') + user = request.args.get('redirect') + key = request.args.get('key') + redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', key) + if user: + return redirect(url_for('uuid_management', uuid=uuid_sensor)) + + +# demo function +@app.route('/whois_data') +def whois_data(): + ip = request.args.get('ip') + if is_valid_ip: + return jsonify(get_whois_ouput(ip)) + else: + return 'Invalid IP' + +@app.route('/generate_uuid') +def generate_uuid(): + new_uuid = uuid.uuid4() + return jsonify({'uuid': new_uuid}) + +@app.route('/get_analyser_sample') +def get_analyser_sample(): + type = request.args.get('type') + analyzer_uuid = request.args.get('analyzer_uuid') + max_line_len = request.args.get('max_line_len') + # get max_line_len + if max_line_len is not None and max_line_len!= 'undefined': + try: + max_line_len = int(max_line_len) + except: + max_line_len = default_analyzer_max_line_len + if max_line_len < 1: + max_line_len = default_analyzer_max_line_len + else: + max_line_len = default_analyzer_max_line_len + if is_valid_uuid_v4(analyzer_uuid): + list_queue = redis_server_analyzer.lrange('analyzer:{}:{}'.format(type, analyzer_uuid), 0 ,10) + list_queue_res = [] + for res in list_queue: + #limit line len + if len(res) > max_line_len: + res = '{} [...]'.format(res[:max_line_len]) + list_queue_res.append('{}\n'.format(res)) + return jsonify(''.join(list_queue_res)) + else: + return jsonify('Incorrect UUID') + +@app.route('/get_uuid_type_history_json') +def get_uuid_type_history_json(): + uuid_sensor = request.args.get('uuid_sensor') + if is_valid_uuid_v4(uuid_sensor): + num_day_type = 7 + date_range = get_substract_date_range(num_day_type) + type_history = [] + range_decoder = [] + all_type = set() + for date in date_range: + type_day = redis_server_metadata.zrange('stat_uuid_type:{}:{}'.format(date, uuid_sensor), 0, -1, withscores=True) + for type in type_day: + all_type.add(type[0]) + range_decoder.append((date, type_day)) + + default_dict_type = {} + for type in all_type: + default_dict_type[type] = 0 + for row in range_decoder: + day_type = default_dict_type.copy() + date = row[0] + day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] + for type in row[1]: + day_type[type[0]]= type[1] + type_history.append(day_type) + + return jsonify(type_history) + else: + return jsonify('Incorrect UUID') + + + + if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True) diff --git a/server/web/static/img/cef.png b/server/web/static/img/cef.png new file mode 100644 index 0000000..a4a8e4f Binary files /dev/null and b/server/web/static/img/cef.png differ diff --git a/server/web/static/img/circl.png b/server/web/static/img/circl.png new file mode 100644 index 0000000..5d13dee Binary files /dev/null and b/server/web/static/img/circl.png differ diff --git a/server/web/templates/404.html b/server/web/templates/404.html new file mode 100644 index 0000000..432b906 --- /dev/null +++ b/server/web/templates/404.html @@ -0,0 +1,70 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + + + +
+ 
+                                __    __   ______   __    __
+                               /  |  /  | /      \ /  |  /  |
+                               $$ |  $$ |/$$$$$$  |$$ |  $$ |
+                               $$ |__$$ |$$$  \$$ |$$ |__$$ |
+                               $$    $$ |$$$$  $$ |$$    $$ |
+                               $$$$$$$$ |$$ $$ $$ |$$$$$$$$ |
+                                     $$ |$$ \$$$$ |      $$ |
+                                     $$ |$$   $$$/       $$ |
+       _______   __    __            $$/  $$$$$$/        $$/                                __
+      /       \ /  |  /  |                                                                 /  |
+      $$$$$$$  |$$ |  $$ |        ______    ______    ______      __   ______    _______  _$$ |_
+      $$ |  $$ |$$ |__$$ |       /      \  /      \  /      \    /  | /      \  /       |/ $$   |
+      $$ |  $$ |$$    $$ |      /$$$$$$  |/$$$$$$  |/$$$$$$  |   $$/ /$$$$$$  |/$$$$$$$/ $$$$$$/
+      $$ |  $$ |$$$$$$$$ |      $$ |  $$ |$$ |  $$/ $$ |  $$ |   /  |$$    $$ |$$ |        $$ | __
+      $$ |__$$ |      $$ |      $$ |__$$ |$$ |      $$ \__$$ |   $$ |$$$$$$$$/ $$ \_____   $$ |/  |
+      $$    $$/       $$ |      $$    $$/ $$ |      $$    $$/    $$ |$$       |$$       |  $$  $$/
+      $$$$$$$/        $$/       $$$$$$$/  $$/        $$$$$$/__   $$ | $$$$$$$/  $$$$$$$/    $$$$/
+                                $$ |                       /  \__$$ |
+                                $$ |                       $$    $$/
+                                $$/                         $$$$$$/
+
+
+
+ + {% include 'navfooter.html' %} + diff --git a/server/web/templates/blacklisted_ip.html b/server/web/templates/blacklisted_ip.html new file mode 100644 index 0000000..49c880d --- /dev/null +++ b/server/web/templates/blacklisted_ip.html @@ -0,0 +1,198 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + + + +
+
+
+ Blacklisted IP +
+
+ +
+
+ + + + + + + + + {% for ip in list_blacklisted_ip_1 %} + + + + + {% endfor %} + +
IPUnblacklist IP
{{ip}} + + + +
+
+
+
+
+
Blacklist IP
+ +
+ {%if blacklisted_ip==2 %} + This IP is already blacklisted + {% else %} + Incorrect IP address + {% endif %} +
+
+ IP Blacklisted +
+ +
+
+
+
+
Unblacklist IP
+ +
+ {%if unblacklisted_ip==2 %} + This IP is not blacklisted + {% else %} + Incorrect IP address + {% endif %} +
+
+ IP Unblacklisted +
+ +
+
+
+
+ + + + + + + + + {% for ip in list_blacklisted_ip_2 %} + + + + + {% endfor %} + +
IPUnblacklist IP
{{ip}} + + + +
+
+
+
+
+ +
+ +
+ +
+ + {% include 'navfooter.html' %} + + + diff --git a/server/web/templates/blacklisted_uuid.html b/server/web/templates/blacklisted_uuid.html new file mode 100644 index 0000000..22eb5fa --- /dev/null +++ b/server/web/templates/blacklisted_uuid.html @@ -0,0 +1,198 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + + + +
+
+
+ Blacklisted UUID +
+
+ +
+
+ + + + + + + + + {% for uuid in list_blacklisted_uuid_1 %} + + + + + {% endfor %} + +
UUIDUnblacklist UUID
{{uuid}} + + + +
+
+
+
+
+
Blacklist UUID
+ +
+ {%if blacklisted_uuid==2 %} + This UUID is already blacklisted + {% else %} + Incorrect UUID + {% endif %} +
+
+ UUID Blacklisted +
+ +
+
+
+
+
Unblacklist UUID
+ +
+ {%if unblacklisted_uuid==2 %} + This UUID is not blacklisted + {% else %} + Incorrect UUID + {% endif %} +
+
+ UUID Unblacklisted +
+ +
+
+
+
+ + + + + + + + + {% for uuid in list_blacklisted_uuid_2 %} + + + + + {% endfor %} + +
UUIDUnblacklist UUID
{{uuid}} + + + +
+
+
+
+
+ +
+ +
+ +
+ + {% include 'navfooter.html' %} + + + diff --git a/server/web/templates/index.html b/server/web/templates/index.html index 257046f..0d3eab0 100644 --- a/server/web/templates/index.html +++ b/server/web/templates/index.html @@ -2,10 +2,15 @@ + D4-Project + + + + @@ -45,24 +50,65 @@ + -
+ +
-
-
-
+
+
+ UUID +
+
+
+
+
+ {{date}} +
+
-
-
+
+
+ Types +
+
+
+
+
+ {{date}} +
+
+ + + +
+ + {% include 'navfooter.html' %} + + + + + + + + + + + +
+
+
+
+
+ + +
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+ + {% for row_uuid in status_daily_uuid %} +
+ +
+ UUID: {{row_uuid['uuid']}} +
+ +
+
+
+
+ First Seen +
+
+

{{row_uuid['first_seen_gmt']}} - ({{row_uuid['first_seen']}})

+
+
+
+
+ Last Seen +
+
+

{{row_uuid['last_seen_gmt']}} - ({{row_uuid['last_seen']}})

+
+
+
+ {% if not row_uuid['Error'] %} +
+ Status +
+
+

OK

+ {% else %} +
+ Status +
+
+

{{row_uuid['Error']}}

+ {% endif %} + {% if row_uuid['active_connection'] %} +
+ Connected +
+ {% endif %} +
+
+
+
+
+ {% endfor %} + + {% include 'navfooter.html' %} + diff --git a/server/web/templates/server_management.html b/server/web/templates/server_management.html new file mode 100644 index 0000000..028fa76 --- /dev/null +++ b/server/web/templates/server_management.html @@ -0,0 +1,464 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + +
+
+
+ Blacklist IP +
+
+
+
+
+
Blacklist IP
+ +
+ {%if blacklisted_ip==2 %} + This IP is already blacklisted + {% else %} + Incorrect IP address + {% endif %} +
+
+ IP Blacklisted +
+ +
+
+
+
+
Manage IP Blacklist
+ + + +
+
+
+
+
Unblacklist IP
+ +
+ {%if unblacklisted_ip==2 %} + This IP is not blacklisted + {% else %} + Incorrect IP address + {% endif %} +
+
+ IP Unblacklisted +
+ +
+
+
+
+
+ +
+ +
+
+ Blacklist UUID +
+
+
+
+
+
Blacklist UUID
+ +
+ {%if blacklisted_uuid==2 %} + This UUID is already blacklisted + {% else %} + Incorrect UUID + {% endif %} +
+
+ UUID Blacklisted +
+ +
+
+
+
+
Manage UUID Blacklist
+ + + +
+
+
+
+
Unblacklist UUID
+ +
+ {%if unblacklisted_uuid==2 %} + This UUID is not Blacklisted + {% else %} + Incorrect UUID + {% endif %} +
+
+ UUID Unblacklisted +
+ +
+
+
+
+
+
+ +
+
+
+ Header Accepted Types +
+
+ +
+
+ + + + + + + + + + {% for type in list_accepted_types %} + + + + + + {% endfor %} + +
TypeDescriptionRemove Type
{{type['id']}}{{type['description']}} + + + +
+ +
+ + + + + + + + + + {% for type in list_accepted_extended_types %} + + + + + + {% endfor %} + +
Type NameDescriptionRemove Type
{{type['name']}}{{type['description']}} + + + +
+
+ +
+
+
+
+
Add New Types
+ + + +
+
+
+
+ +
+
+ +
+ + +
+
+
+ Analyzer Management +
+
+ +
+
+ + + + + + + + + + + + {% for type in list_accepted_types %} + {% if type['list_analyzer_uuid'] %} + {% for analyzer in type['list_analyzer_uuid'] %} + + + + + + + + {% endfor %} + {% endif %} + {% endfor %} + +
Typeuuidlast updatedChange max size limitAnalyzer Queue
{{type['id']}} +
+ {{analyzer['uuid']}} + + + +
+ {%if analyzer['description']%} +
{{analyzer['description']}}
+ {%endif%} + 		{{analyzer['last_updated']}} +
+ + +
+ 		+ + + + +
+ +
+ + + + + + + + + + + + {% for type in list_accepted_extended_types %} + {% if type['list_analyzer_uuid'] %} + {% for analyzer in type['list_analyzer_uuid'] %} + + + + + + + + {% endfor %} + {% endif %} + {% endfor %} + +
Type Nameuuidlast updatedChange max size limitAnalyzer Queue
{{type['name']}} +
+ {{analyzer['uuid']}} + + + +
+ {%if analyzer['description']%} +
{{analyzer['description']}}
+ {%endif%} + 		{{analyzer['last_updated']}} +
+ + +
+ 		+ + + + +
+
+ +
+
+
+
+
Add New Analyzer Queue
+ + +
+
+ +
+ +
+ + +
+
+
+
+ +
+
+ +
+ + + + {% include 'navfooter.html' %} + + + diff --git a/server/web/templates/uuid_management.html b/server/web/templates/uuid_management.html new file mode 100644 index 0000000..e524664 --- /dev/null +++ b/server/web/templates/uuid_management.html @@ -0,0 +1,403 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + + + +
+
+ UUID: {{uuid_sensor}} +
+
+
+
+
+ First Seen +
+
+

{{data_uuid['first_seen_gmt']}} - ({{data_uuid['first_seen']}})

+
+
+
+
+ Last Seen +
+
+

{{data_uuid['last_seen_gmt']}} - ({{data_uuid['last_seen']}})

+
+
+
+ {% if not data_uuid['Error'] %} +
+ Status +
+
+

OK

+ {% else %} +
+ Status +
+
+

{{data_uuid['Error']}}

+ {% endif %} + {% if active_connection %} +
+ Connected +
+ + {% endif %} +
+
+
+ +
+
+ +
+
+
+
Change Stream Max Size
+ {% if not data_uuid['blacklisted_uuid'] and not data_uuid['blacklisted_ip_by_uuid'] %} + + + {% else %} + + + {% endif %} +
+
+
+
+
UUID Blacklist
+ {% if not data_uuid['blacklisted_uuid'] %} + + + + {% else %} + + + + {% endif %} +
+
+
+
+
Blacklist IP Using This UUID
+ {% if not data_uuid['blacklisted_ip_by_uuid'] %} + + + + {% else %} + + + + {% endif %} +
+
+
+
+
Change UUID Key
+ + +
+
+
+ +
+
+
+ Types Used: +
+
+
+
+ + + + + + + + + + {% for type in uuid_all_type %} + + + + + + {% endfor %} + +
Typefirst seenlast seen
{{type['type']}}{{type['first_seen']}}{{type['last_seen']}}
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+ Last IP Used: +
+
    + {%for row in all_ip%} +
  • + {{row['ip']}} - {{row['datetime']}} +
    • + {%endfor%} +
+
+
+
+
+
+ Whois Info: +
+ 
+
+
+
+
+ + {% include 'navfooter.html' %} + + + + + diff --git a/server/web/update_web.sh b/server/web/update_web.sh index 1262161..ca25549 100755 --- a/server/web/update_web.sh +++ b/server/web/update_web.sh @@ -5,6 +5,7 @@ set -e BOOTSTRAP_VERSION='4.2.1' FONT_AWESOME_VERSION='4.7.0' D3_JS_VERSION='4.13.0' +D3_JS_VERSIONv5='5.9.2' if [ ! -d static/css ]; then mkdir static/css @@ -12,42 +13,55 @@ fi if [ ! -d static/js ]; then mkdir static/js fi +if [ ! -d static/json ]; then + mkdir static/json +fi rm -rf temp mkdir temp +mkdir temp/d3v5/ + wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip -#wget https://github.com/FortAwesome/Font-Awesome/archive/v${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip +wget https://github.com/FortAwesome/Font-Awesome/archive/v${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip +wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSIONv5}/d3.zip -O temp/d3v5/d3_${D3_JS_VERSIONv5}.zip +wget https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip # dateRangePicker -#wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip -#wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip +wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip +wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip unzip temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/ -#unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ +unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ unzip temp/d3_${D3_JS_VERSION}.zip -d temp/ +unzip temp/d3v5/d3_${D3_JS_VERSIONv5}.zip -d temp/d3v5/ +unzip temp/popper.zip -d temp/ -#unzip temp/moment_2.22.2.zip -d temp/ -#unzip temp/daterangepicker_v0.18.0.zip -d temp/ +unzip temp/moment_2.22.2.zip -d temp/ +unzip temp/daterangepicker_v0.18.0.zip -d temp/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/ +mv temp/popper.js-1.14.3/dist/umd/popper.min.js ./static/js/ +mv temp/popper.js-1.14.3/dist/umd/popper.min.js.map ./static/js/ -#mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome +mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome -#rm -rf ./static/fonts/ ./static/font-awesome/ -#mv temp/font-awesome/ ./static/ +rm -rf ./static/fonts/ ./static/font-awesome/ +mv temp/font-awesome/ ./static/ -#mv temp/jquery-date-range-picker-0.18.0/dist/daterangepicker.min.css ./static/css/ +mv temp/jquery-date-range-picker-0.18.0/dist/daterangepicker.min.css ./static/css/ mv temp/d3.min.js ./static/js/ -#mv temp/moment-2.22.2/min/moment.min.js ./static/js/ -#mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./static/js/ +cp temp/d3v5/d3.min.js ./static/js/d3v5.min.js + +mv temp/moment-2.22.2/min/moment.min.js ./static/js/ +mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./static/js/ rm -rf temp @@ -55,7 +69,11 @@ JQVERSION="3.3.1" wget http://code.jquery.com/jquery-${JQVERSION}.min.js -O ./static/js/jquery.js #Ressources for dataTable -wget https://cdn.datatables.net/v/bs4/dt-1.10.18/datatables.min.css -O ./static/css/dataTables.bootstrap.css -wget https://cdn.datatables.net/v/bs4/dt-1.10.18/datatables.min.js -O ./static/js/dataTables.bootstrap.js +wget https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css +wget https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js +wget https://cdn.datatables.net/1.10.18/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js + +#Update json +wget https://raw.githubusercontent.com/D4-project/architecture/master/format/type.json -O ./static/json/type.json rm -rf temp diff --git a/server/workers/workers_1/file_compressor.py b/server/workers/workers_1/file_compressor.py new file mode 100755 index 0000000..3406566 --- /dev/null +++ b/server/workers/workers_1/file_compressor.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import gzip +import redis +import shutil +import datetime + +import signal + +class GracefulKiller: + kill_now = False + def __init__(self): + signal.signal(signal.SIGINT, self.exit_gracefully) + signal.signal(signal.SIGTERM, self.exit_gracefully) + + def exit_gracefully(self,signum, frame): + self.kill_now = True + +def compress_file(file_full_path, session_uuid,i=0): + redis_server_stream.set('data_in_process:{}'.format(session_uuid), file_full_path) + if i==0: + compressed_filename = '{}.gz'.format(file_full_path) + else: + compressed_filename = '{}.{}.gz'.format(file_full_path, i) + if os.path.isfile(compressed_filename): + compress_file(file_full_path, session_uuid, i+1) + else: + with open(file_full_path, 'rb') as f_in: + with gzip.open(compressed_filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + try: + os.remove(file_full_path) + except FileNotFoundError: + pass + # save full path in anylyzer queue + for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)): + analyzer_uuid = analyzer_uuid.decode() + redis_server_analyzer.lpush('analyzer:{}:{}'.format(type, analyzer_uuid), compressed_filename) + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time()) + analyser_queue_max_size = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if analyser_queue_max_size is None: + analyser_queue_max_size = analyzer_list_max_default_size + redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) + + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +host_redis_metadata = "localhost" +port_redis_metadata = 6380 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) + +redis_server_metadata = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=0) + +redis_server_analyzer = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=2) + +type = 1 +sleep_time = 300 + +analyzer_list_max_default_size = 10000 + +if __name__ == "__main__": + killer = GracefulKiller() + + if len(sys.argv) != 4: + print('usage:', 'Worker.py', 'session_uuid', 'tcpdump', 'date') + exit(1) + + # TODO sanityse input + session_uuid = sys.argv[1] + directory_data_uuid = sys.argv[2] + date = sys.argv[3] + + worker_data_directory = os.path.join(directory_data_uuid, date[0:4], date[4:6], date[6:8]) + full_datetime = datetime.datetime.now().strftime("%Y%m%d%H") + + current_file = None + time_change = False + + while True: + if killer.kill_now: + break + + new_date = datetime.datetime.now().strftime("%Y%m%d") + + # get all directory files + all_files = os.listdir(worker_data_directory) + not_compressed_file = [] + # filter: get all not compressed files + for file in all_files: + if file.endswith('.cap'): + not_compressed_file.append(os.path.join(worker_data_directory, file)) + + if not_compressed_file: + ### check time-change (minus one hour) ### + new_full_datetime = datetime.datetime.now().strftime("%Y%m%d%H") + if new_full_datetime < full_datetime: + # sort list, last modified + not_compressed_file.sort(key=os.path.getctime) + else: + # sort list + not_compressed_file.sort() + ### ### + + # new day + if date != new_date: + # compress all file + for file in not_compressed_file: + if killer.kill_now: + break + compress_file(file, session_uuid) + # reset file tracker + current_file = None + date = new_date + # update worker_data_directory + worker_data_directory = os.path.join(directory_data_uuid, date[0:4], date[4:6], date[6:8]) + # restart + continue + + # file used by tcpdump + max_file = not_compressed_file[-1] + full_datetime = new_full_datetime + + # Init: set current_file + if not current_file: + current_file = max_file + #print('max_file set: {}'.format(current_file)) + + # new file created + if max_file != current_file: + + # get all previous files + for file in not_compressed_file: + if file != max_file: + if killer.kill_now: + break + #print('new file: {}'.format(file)) + compress_file(file, session_uuid) + + # update current_file tracker + current_file = max_file + + if killer.kill_now: + break + + time.sleep(sleep_time) diff --git a/server/workers/workers_1/worker.py b/server/workers/workers_1/worker.py index 0d89030..a643ed5 100755 --- a/server/workers/workers_1/worker.py +++ b/server/workers/workers_1/worker.py @@ -3,16 +3,18 @@ import os import sys import time +import gzip import redis -import subprocess - +import shutil import datetime +import subprocess +import configparser def data_incorrect_format(stream_name, session_uuid, uuid): - redis_server_stream.sadd('Error:IncorrectType:{}'.format(type), session_uuid) + redis_server_stream.sadd('Error:IncorrectType', session_uuid) redis_server_metadata.hset('metadata_uuid:{}'.format(uuid), 'Error', 'Error: Type={}, Incorrect file format'.format(type)) clean_stream(stream_name, session_uuid) - print('Incorrect format') + print('Incorrect format, uuid={}'.format(uuid)) sys.exit(1) def clean_stream(stream_name, session_uuid): @@ -22,6 +24,28 @@ def clean_stream(stream_name, session_uuid): redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid) redis_server_stream.delete(stream_name) +def compress_file(file_full_path, i=0): + if i==0: + compressed_filename = '{}.gz'.format(file_full_path) + else: + compressed_filename = '{}.{}.gz'.format(file_full_path, i) + if os.path.isfile(compressed_filename): + compress_file(file_full_path, i+1) + else: + with open(file_full_path, 'rb') as f_in: + with gzip.open(compressed_filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + os.remove(file_full_path) + # save full path in anylyzer queue + for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)): + analyzer_uuid = analyzer_uuid.decode() + redis_server_analyzer.lpush('analyzer:{}:{}'.format(type, analyzer_uuid), compressed_filename) + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time()) + analyser_queue_max_size = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if analyser_queue_max_size is None: + analyser_queue_max_size = analyzer_list_max_default_size + redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) + host_redis_stream = "localhost" port_redis_stream = 6379 @@ -38,10 +62,31 @@ redis_server_metadata = redis.StrictRedis( port=port_redis_metadata, db=0) +redis_server_analyzer = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=2) + +# get file config +config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') +config_server = configparser.ConfigParser() +config_server.read(config_file_server) + +# get data directory +use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +# check if field is None +if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') +else: + data_directory = config_server['Save_Directories'].get('save_directory') + + type = 1 tcp_dump_cycle = '300' stream_buffer = 100 +analyzer_list_max_default_size = 10000 + id_to_delete = [] if __name__ == "__main__": @@ -58,11 +103,12 @@ if __name__ == "__main__": if res: uuid = res[0][1][0][1][b'uuid'].decode() date = datetime.datetime.now().strftime("%Y%m%d") - tcpdump_path = os.path.join('../../data', uuid, str(type)) + tcpdump_path = os.path.join(data_directory, uuid, str(type)) + full_tcpdump_path = os.path.join(data_directory, uuid, str(type)) rel_path = os.path.join(tcpdump_path, date[0:4], date[4:6], date[6:8]) if not os.path.isdir(rel_path): os.makedirs(rel_path) - print('---- worker launched, uuid={} session_uuid={}'.format(uuid, session_uuid)) + print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) else: sys.exit(1) print('Incorrect message') @@ -72,6 +118,8 @@ if __name__ == "__main__": process = subprocess.Popen(["tcpdump", '-n', '-r', '-', '-G', tcp_dump_cycle, '-w', '{}/%Y/%m/%d/{}-%Y-%m-%d-%H%M%S.cap'.format(tcpdump_path, uuid)], stdin=subprocess.PIPE, stderr=subprocess.PIPE) nb_save = 0 + process_compressor = subprocess.Popen(['./file_compressor.py', session_uuid, full_tcpdump_path, date]) + while True: res = redis_server_stream.xread({stream_name: id}, count=1) @@ -97,6 +145,8 @@ if __name__ == "__main__": Error_message = process.stderr.read() if Error_message == b'tcpdump: unknown file format\n': data_incorrect_format(stream_name, session_uuid, uuid) + elif Error_message: + print(Error_message) #print(process.stdout.read()) nb_save += 1 @@ -108,24 +158,44 @@ if __name__ == "__main__": nb_save = 0 else: - # sucess, all data are saved + # success, all data are saved if redis_server_stream.sismember('ended_session', session_uuid): out, err = process.communicate(timeout= 0.5) - #print(out) + #if out: + # print(out) if err == b'tcpdump: unknown file format\n': data_incorrect_format(stream_name, session_uuid, uuid) elif err: print(err) + # close child + try: + process_compressor.communicate(timeout= 0.5) + except subprocess.TimeoutExpired: + process_compressor.kill() + ### compress all files ### + date = datetime.datetime.now().strftime("%Y%m%d") + worker_data_directory = os.path.join(full_tcpdump_path, date[0:4], date[4:6], date[6:8]) + all_files = os.listdir(worker_data_directory) + all_files.sort() + if all_files: + for file in all_files: + if file.endswith('.cap'): + full_path = os.path.join(worker_data_directory, file) + if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path: + compress_file(full_path) + ### ### + #print(process.stderr.read()) redis_server_stream.srem('ended_session', session_uuid) redis_server_stream.srem('session_uuid:{}'.format(type), session_uuid) redis_server_stream.srem('working_session_uuid:{}'.format(type), session_uuid) redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid) redis_server_stream.delete(stream_name) + redis_server_stream.delete('data_in_process:{}'.format(session_uuid)) # make sure that tcpdump can save all datas time.sleep(10) - print('---- tcpdump DONE, uuid={} session_uuid={}'.format(uuid, session_uuid)) + print('---- tcpdump DONE, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) sys.exit(0) else: time.sleep(10) diff --git a/server/workers/workers_2/file_compressor.py b/server/workers/workers_2/file_compressor.py new file mode 100755 index 0000000..3406566 --- /dev/null +++ b/server/workers/workers_2/file_compressor.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import gzip +import redis +import shutil +import datetime + +import signal + +class GracefulKiller: + kill_now = False + def __init__(self): + signal.signal(signal.SIGINT, self.exit_gracefully) + signal.signal(signal.SIGTERM, self.exit_gracefully) + + def exit_gracefully(self,signum, frame): + self.kill_now = True + +def compress_file(file_full_path, session_uuid,i=0): + redis_server_stream.set('data_in_process:{}'.format(session_uuid), file_full_path) + if i==0: + compressed_filename = '{}.gz'.format(file_full_path) + else: + compressed_filename = '{}.{}.gz'.format(file_full_path, i) + if os.path.isfile(compressed_filename): + compress_file(file_full_path, session_uuid, i+1) + else: + with open(file_full_path, 'rb') as f_in: + with gzip.open(compressed_filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + try: + os.remove(file_full_path) + except FileNotFoundError: + pass + # save full path in anylyzer queue + for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)): + analyzer_uuid = analyzer_uuid.decode() + redis_server_analyzer.lpush('analyzer:{}:{}'.format(type, analyzer_uuid), compressed_filename) + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time()) + analyser_queue_max_size = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if analyser_queue_max_size is None: + analyser_queue_max_size = analyzer_list_max_default_size + redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) + + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +host_redis_metadata = "localhost" +port_redis_metadata = 6380 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) + +redis_server_metadata = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=0) + +redis_server_analyzer = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=2) + +type = 1 +sleep_time = 300 + +analyzer_list_max_default_size = 10000 + +if __name__ == "__main__": + killer = GracefulKiller() + + if len(sys.argv) != 4: + print('usage:', 'Worker.py', 'session_uuid', 'tcpdump', 'date') + exit(1) + + # TODO sanityse input + session_uuid = sys.argv[1] + directory_data_uuid = sys.argv[2] + date = sys.argv[3] + + worker_data_directory = os.path.join(directory_data_uuid, date[0:4], date[4:6], date[6:8]) + full_datetime = datetime.datetime.now().strftime("%Y%m%d%H") + + current_file = None + time_change = False + + while True: + if killer.kill_now: + break + + new_date = datetime.datetime.now().strftime("%Y%m%d") + + # get all directory files + all_files = os.listdir(worker_data_directory) + not_compressed_file = [] + # filter: get all not compressed files + for file in all_files: + if file.endswith('.cap'): + not_compressed_file.append(os.path.join(worker_data_directory, file)) + + if not_compressed_file: + ### check time-change (minus one hour) ### + new_full_datetime = datetime.datetime.now().strftime("%Y%m%d%H") + if new_full_datetime < full_datetime: + # sort list, last modified + not_compressed_file.sort(key=os.path.getctime) + else: + # sort list + not_compressed_file.sort() + ### ### + + # new day + if date != new_date: + # compress all file + for file in not_compressed_file: + if killer.kill_now: + break + compress_file(file, session_uuid) + # reset file tracker + current_file = None + date = new_date + # update worker_data_directory + worker_data_directory = os.path.join(directory_data_uuid, date[0:4], date[4:6], date[6:8]) + # restart + continue + + # file used by tcpdump + max_file = not_compressed_file[-1] + full_datetime = new_full_datetime + + # Init: set current_file + if not current_file: + current_file = max_file + #print('max_file set: {}'.format(current_file)) + + # new file created + if max_file != current_file: + + # get all previous files + for file in not_compressed_file: + if file != max_file: + if killer.kill_now: + break + #print('new file: {}'.format(file)) + compress_file(file, session_uuid) + + # update current_file tracker + current_file = max_file + + if killer.kill_now: + break + + time.sleep(sleep_time) diff --git a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py new file mode 100755 index 0000000..dcd4b77 --- /dev/null +++ b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import json +import gzip +import redis +import shutil +import datetime +import configparser + +DEFAULT_FILE_EXTENSION = 'txt' +DEFAULT_FILE_SEPARATOR = b'\n' +ROTATION_SAVE_CYCLE = 300 # seconds +MAX_BUFFER_LENGTH = 100000 +TYPE = 254 + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) + +host_redis_metadata = "localhost" +port_redis_metadata = 6380 + +redis_server_metadata = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=0) + +redis_server_analyzer = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=2) + +analyzer_list_max_default_size = 10000 + +class MetaTypesDefault: + + def __init__(self, uuid, json_file): + self.uuid = uuid + self.type_name = json_file['type'] + self.save_path = None + self.buffer = b'' + self.file_rotation_mode = True + + # get file config + config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') + config_server = configparser.ConfigParser() + config_server.read(config_file_server) + # get data directory + use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') + # check if field is None + if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') + else: + data_directory = config_server['Save_Directories'].get('save_directory') + self.data_directory = data_directory + + self.parse_json(json_file) + + def test(self): + print('class: MetaTypesDefault') + + ######## JSON PARSER ######## + def parse_json(self, json_file): + self.file_rotation = False + self.file_separator = b'\n' + self.filename = b''.join([self.type_name.encode(), b'.txt']) + + ######## PROCESS FUNCTIONS ######## + def process_data(self, data): + # save data on disk + self.save_rotate_file(data) + + ######## CORE FUNCTIONS ######## + + def check_json_file(self, json_file): + # the json object must contain a type field + if "type" in json_file: + return True + else: + return False + + def save_json_file(self, json_file, save_by_uuid=True): + self.set_last_time_saved(time.time()) #time_file + self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) #date_file + # update save path + self.set_save_path( os.path.join(self.get_save_dir(save_by_uuid=save_by_uuid), self.get_filename(file_extention='json', save_by_uuid=save_by_uuid)) ) + # save json + with open(self.get_save_path(), 'w') as f: + f.write(json.dumps(json_file)) + # update save path for 254 files type + if self.is_file_rotation_mode(): + self.set_save_path( os.path.join(self.get_save_dir(), self.get_filename()) ) + + + def save_rotate_file(self, data): + if not self.get_file_rotation(): + new_date = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + # check if a new file rotation is needed # # TODO: change ROTATION_SAVE_CYCLE + if ( new_date[0:8] != self.get_last_saved_date()[0:8] ) or ( int(time.time()) - self.get_last_time_saved() > ROTATION_SAVE_CYCLE ): + self.set_rotate_file(True) + + # rotate file + if self.get_file_rotation(): + # init save path + if self.get_save_path() is None: + self.set_last_time_saved(time.time()) + self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + # update save path + self.set_save_path( os.path.join(self.get_save_dir(), self.get_filename()) ) + + # rotate file + if self.get_file_separator() in data: + end_file, start_new_file = data.rsplit(self.get_file_separator(), maxsplit=1) + # save end of file + with open(self.get_save_path(), 'ab') as f: + f.write(end_file) + self.compress_file(self.get_save_path()) + + # set last saved date/time + self.set_last_time_saved(time.time()) + self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + # update save path + self.set_save_path( os.path.join(self.get_save_dir(), self.get_filename()) ) + + # save start of new file + if start_new_file != b'': + with open(self.get_save_path(), 'ab') as f: + f.write(start_new_file) + # end of rotation + self.set_rotate_file(False) + + # wait file separator + else: + with open(self.get_save_path(), 'ab') as f: + f.write(data) + else: + # save file + with open(self.get_save_path(), 'ab') as f: + f.write(data) + + def reconstruct_data(self, data): + # save data in buffer + self.add_to_buffer(data) + data = self.get_buffer() + + # end of element found in data + if self.get_file_separator() in data: + # empty buffer + self.reset_buffer() + all_line = data.split(self.get_file_separator()) + for reconstructed_data in all_line[:-1]: + self.handle_reconstructed_data(reconstructed_data) + + # save incomplete element in buffer + if all_line[-1] != b'': + self.add_to_buffer(all_line[-1]) + # no elements + else: + # force file_separator when max buffer size is reached + if self.get_size_buffer() > MAX_BUFFER_LENGTH: + print('Error, infinite loop, max buffer length reached') + self.add_to_buffer(self.get_file_separator()) + + def handle_reconstructed_data(self, data): + # send data to analyzer + self.send_to_analyzers(data) + + def compress_file(self, file_full_path, i=0): + if i==0: + compressed_filename = '{}.gz'.format(file_full_path) + else: + compressed_filename = '{}.{}.gz'.format(file_full_path, i) + if os.path.isfile(compressed_filename): + self.compress_file(file_full_path, i+1) + else: + with open(file_full_path, 'rb') as f_in: + with gzip.open(compressed_filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + os.remove(file_full_path) + + def send_to_analyzers(self, data_to_send): + ## save full path in anylyzer queue + for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}:{}'.format(TYPE, self.get_type_name())): + analyzer_uuid = analyzer_uuid.decode() + redis_server_analyzer.lpush('analyzer:{}:{}'.format(self.get_type_name(), analyzer_uuid), data_to_send) + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time()) + analyser_queue_max_size = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if analyser_queue_max_size is None: + analyser_queue_max_size = analyzer_list_max_default_size + redis_server_analyzer.ltrim('analyzer:{}:{}'.format(self.get_type_name(), analyzer_uuid), 0, analyser_queue_max_size) + + ######## GET FUNCTIONS ######## + + def get_type_name(self): + return self.type_name + + def get_file_separator(self): + return self.file_separator + + def get_uuid(self): + return self.uuid + + def get_buffer(self): + return self.buffer + + def get_size_buffer(self): + return len(self.buffer) + + def get_filename(self, file_extention=None, save_by_uuid=False): + if file_extention is None: + file_extention = DEFAULT_FILE_EXTENSION + # File Rotation, : data//254//// + if self.is_file_rotation_mode() or save_by_uuid: + return '{}-{}-{}-{}-{}.{}'.format(self.uuid, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day(), self.get_last_saved_hour_minute(), file_extention) + + def get_data_save_directory(self): + return self.data_directory + + def get_save_dir(self, save_by_uuid=False): + # File Rotation, save data in directory: data//254//// + if self.is_file_rotation_mode() or save_by_uuid: + data_directory_uuid_type = os.path.join(self.get_data_save_directory(), self.get_uuid(), str(TYPE)) + return os.path.join(data_directory_uuid_type, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day() , self.type_name) + + # data save in the same directory + else: + save_dir = os.path.join(self.get_data_save_directory(), 'datas', self.get_type_name()) + if not os.path.isdir(save_dir): + os.makedirs(save_dir) + return save_dir + + def get_save_path(self): + return self.save_path + + def is_empty_buffer(self): + if self.buffer==b'': + return True + else: + return False + + def is_file_rotation_mode(self): + if self.file_rotation_mode: + return True + else: + return False + + def get_file_rotation(self): + return self.file_rotation + + def get_last_time_saved(self): + return self.last_time_saved + + def get_last_saved_date(self): + return self.last_saved_date + + def get_last_saved_year(self): + return self.last_saved_date[0:4] + + def get_last_saved_month(self): + return self.last_saved_date[4:6] + + def get_last_saved_day(self): + return self.last_saved_date[6:8] + + def get_last_saved_hour_minute(self): + return self.last_saved_date[8:14] + + ######## SET FUNCTIONS ######## + + def reset_buffer(self): + self.buffer = b'' + + def set_buffer(self, data): + self.buffer = data + + def add_to_buffer(self, data): + self.buffer = b''.join([self.buffer, data]) + + def set_rotate_file(self, boolean_value): + self.file_rotation = boolean_value + + def set_rotate_file_mode(self, boolean_value): + self.file_rotation_mode = boolean_value + + def set_last_time_saved(self, value_time): + self.last_time_saved = int(value_time) + + def set_last_saved_date(self, date): + self.last_saved_date = date + + def set_save_path(self, save_path): + dir_path = os.path.dirname(save_path) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + self.save_path = save_path diff --git a/server/workers/workers_2/meta_types_modules/ja3-jl/ja3-jl.py b/server/workers/workers_2/meta_types_modules/ja3-jl/ja3-jl.py new file mode 100755 index 0000000..7552413 --- /dev/null +++ b/server/workers/workers_2/meta_types_modules/ja3-jl/ja3-jl.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import json +import redis +import datetime +import hashlib +import binascii +import redis +import pdb + +from meta_types_modules.MetaTypesDefault import MetaTypesDefault + +class TypeHandler(MetaTypesDefault): + + def __init__(self, uuid, json_file): + super().__init__(uuid, json_file) + self.set_rotate_file_mode(False) + + def process_data(self, data): + self.reconstruct_data(data) + + def handle_reconstructed_data(self, data): + self.set_last_time_saved(time.time()) + self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + + # Create folders + cert_save_dir = os.path.join(self.get_save_dir(), 'certs') + jsons_save_dir = os.path.join(self.get_save_dir(), 'jsons') + if not os.path.isdir(cert_save_dir): + os.makedirs(cert_save_dir) + if not os.path.isdir(jsons_save_dir): + os.makedirs(jsons_save_dir) + + # Extract certificates from json + mtjson = json.loads(data.decode()) + for certificate in mtjson["Certificates"] or []: + cert = binascii.a2b_base64(certificate["Raw"]) + # one could also load this cert with + # xcert = x509.load_der_x509_certificate(cert, default_backend()) + m = hashlib.sha1() + m.update(cert) + cert_path = os.path.join(cert_save_dir, m.hexdigest()+'.crt') + # write unique certificate der file to disk + with open(cert_path, 'w+b') as c: + c.write(cert) + + # write json file to disk + jsons_path = os.path.join(jsons_save_dir, mtjson["Timestamp"]+'.json') + with open(jsons_path, 'w') as j: + j.write(data.decode()) + # Send data to Analyszer + self.send_to_analyzers(jsons_path) + + + def test(self): + print('Class: ja3-jl') diff --git a/server/workers/workers_2/worker.py b/server/workers/workers_2/worker.py new file mode 100755 index 0000000..3eee594 --- /dev/null +++ b/server/workers/workers_2/worker.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import json +import redis + +import datetime + +from meta_types_modules import MetaTypesDefault + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) + +host_redis_metadata = "localhost" +port_redis_metadata = 6380 + +redis_server_metadata = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=0) + +type_meta_header = 2 +type_defined = 254 +max_buffer_length = 100000 +rotation_save_cycle = 10 #seconds + +json_file_name = 'meta_json.json' + +def get_class( package_class ): + parts = package_class.split('.') + module = ".".join(parts[:-1]) + mod = __import__( module ) + for comp in parts[1:]: + mod = getattr(mod, comp) + return mod + +def check_default_json_file(json_file): + # the json object must contain a type field + if "type" in json_file: + return True + else: + return False + +def on_error(session_uuid, type_error, message): + redis_server_stream.sadd('Error:IncorrectType', session_uuid) + redis_server_metadata.hset('metadata_uuid:{}'.format(uuid), 'Error', 'Error: Type={}, {}'.format(type_error, message)) + clean_db(session_uuid) + print('Incorrect format') + sys.exit(1) + +def clean_db(session_uuid): + clean_stream(stream_meta_json, type_meta_header, session_uuid) + clean_stream(stream_defined, type_defined, session_uuid) + redis_server_stream.srem('ended_session', session_uuid) + redis_server_stream.srem('working_session_uuid:{}'.format(type_meta_header), session_uuid) + +def clean_stream(stream_name, type, session_uuid): + redis_server_stream.srem('session_uuid:{}'.format(type), session_uuid) + redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid) + redis_server_stream.delete(stream_name) + +if __name__ == "__main__": + + + ###################################################3 + + if len(sys.argv) != 2: + print('usage:', 'Worker.py', 'session_uuid') + exit(1) + + session_uuid = sys.argv[1] + stream_meta_json = 'stream:{}:{}'.format(type_meta_header, session_uuid) + stream_defined = 'stream:{}:{}'.format(type_defined, session_uuid) + + id = '0' + buffer = b'' + + stream_name = stream_meta_json + type = type_meta_header + + # track launched worker + redis_server_stream.sadd('working_session_uuid:{}'.format(type_meta_header), session_uuid) + + # get uuid + res = redis_server_stream.xread({stream_name: id}, count=1) + if res: + uuid = res[0][1][0][1][b'uuid'].decode() + print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) + else: + clean_db(session_uuid) + print('Incorrect Stream, Closing worker: type={} session_uuid={} epoch={}'.format(type, session_uuid, time.time())) + sys.exit(1) + + full_json = None + + # active session + while full_json is None: + + res = redis_server_stream.xread({stream_name: id}, count=1) + if res: + new_id = res[0][1][0][0].decode() + if id != new_id: + id = new_id + data = res[0][1][0][1] + + if id and data: + # remove line from json + data[b'message'] = data[b'message'].replace(b'\n', b'') + + # reconstruct data + if buffer != b'': + data[b'message'] = b''.join([buffer, data[b'message']]) + buffer = b'' + try: + full_json = json.loads(data[b'message'].decode()) + except: + buffer += data[b'message'] + # # TODO: filter too big json + redis_server_stream.xdel(stream_name, id) + + # complete json received + if full_json: + print(full_json) + if check_default_json_file(full_json): + # end type 2 processing + break + # Incorrect Json + else: + on_error(session_uuid, type, 'Incorrect JSON object') + else: + # end session, no json received + if redis_server_stream.sismember('ended_session', session_uuid): + clean_db(session_uuid) + print('---- Incomplete JSON object, DONE, uuid={} session_uuid={}'.format(uuid, session_uuid)) + sys.exit(0) + else: + time.sleep(10) + + # extract/parse JSON + extended_type = full_json['type'] + if not redis_server_metadata.sismember('server:accepted_extended_type', extended_type): + error_mess = 'Unsupported extended_type: {}'.format(extended_type) + on_error(session_uuid, type, error_mess) + clean_db(session_uuid) + sys.exit(1) + + + #### Handle Specific MetaTypes #### + # Use Specific Handler defined + if os.path.isdir(os.path.join('meta_types_modules', extended_type)): + class_type_handler = get_class('meta_types_modules.{}.{}.TypeHandler'.format(extended_type, extended_type)) + type_handler = class_type_handler(uuid, full_json) + # Use Standard Handler + else: + type_handler = MetaTypesDefault.MetaTypesDefault(uuid, full_json) + + #file_separator = type_handler.get_file_separator(self) + #extended_type_name = type_handler.get_file_name() + + # save json on disk + type_handler.save_json_file(full_json) + + # change stream_name/type + stream_name = stream_defined + type = type_defined + id = 0 + buffer = b'' + + type_handler.test() + + # handle 254 type + while True: + res = redis_server_stream.xread({stream_name: id}, count=1) + if res: + new_id = res[0][1][0][0].decode() + if id != new_id: + id = new_id + data = res[0][1][0][1] + + if id and data: + # process 254 data type + type_handler.process_data(data[b'message']) + # remove data from redis stream + redis_server_stream.xdel(stream_name, id) + + else: + # end session, no json received + if redis_server_stream.sismember('ended_session', session_uuid): + clean_db(session_uuid) + print('---- JSON object, DONE, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) + sys.exit(0) + else: + time.sleep(10) diff --git a/server/workers/workers_2/workers_manager.py b/server/workers/workers_2/workers_manager.py new file mode 100755 index 0000000..d66b873 --- /dev/null +++ b/server/workers/workers_2/workers_manager.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import redis +import subprocess + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) +type = 2 + +try: + redis_server_stream.ping() +except redis.exceptions.ConnectionError: + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + sys.exit(1) + +if __name__ == "__main__": + stream_name = 'stream:{}'.format(type) + redis_server_stream.delete('working_session_uuid:{}'.format(type)) + + while True: + for session_uuid in redis_server_stream.smembers('session_uuid:{}'.format(type)): + session_uuid = session_uuid.decode() + if not redis_server_stream.sismember('working_session_uuid:{}'.format(type), session_uuid): + + process = subprocess.Popen(['./worker.py', session_uuid]) + print('Launching new worker{} ... session_uuid={}'.format(type, session_uuid)) + + #print('.') + time.sleep(10) diff --git a/server/workers/workers_4/worker.py b/server/workers/workers_4/worker.py index 436b271..2e044c8 100755 --- a/server/workers/workers_4/worker.py +++ b/server/workers/workers_4/worker.py @@ -6,6 +6,7 @@ import time import redis import datetime +import configparser def data_incorrect_format(session_uuid): print('Incorrect format') @@ -19,6 +20,20 @@ redis_server_stream = redis.StrictRedis( port=port_redis_stream, db=0) +# get file config +config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') +config_server = configparser.ConfigParser() +config_server.read(config_file_server) + +# get data directory +use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +# check if field is None +if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') +else: + data_directory = config_server['Save_Directories'].get('save_directory') + + type = 4 rotation_save_cycle = 300 #seconds @@ -38,13 +53,13 @@ if __name__ == "__main__": if res: date = datetime.datetime.now().strftime("%Y%m%d%H%M%S") uuid = res[0][1][0][1][b'uuid'].decode() - data_rel_path = os.path.join('../../data', uuid, str(type)) + data_rel_path = os.path.join(data_directory, uuid, str(type)) dir_path = os.path.join(data_rel_path, date[0:4], date[4:6], date[6:8]) if not os.path.isdir(dir_path): os.makedirs(dir_path) filename = '{}-{}-{}-{}-{}.dnscap.txt'.format(uuid, date[0:4], date[4:6], date[6:8], date[8:14]) rel_path = os.path.join(dir_path, filename) - print('---- worker launched, uuid={} session_uuid={}'.format(uuid, session_uuid)) + print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) else: sys.exit(1) print('Incorrect message') @@ -98,7 +113,7 @@ if __name__ == "__main__": redis_server_stream.srem('working_session_uuid:{}'.format(type), session_uuid) redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid) redis_server_stream.delete(stream_name) - print('---- dnscap DONE, uuid={} session_uuid={}'.format(uuid, session_uuid)) + print('---- dnscap DONE, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) sys.exit(0) else: time.sleep(10) diff --git a/server/workers/workers_8/worker.py b/server/workers/workers_8/worker.py new file mode 100755 index 0000000..07a406e --- /dev/null +++ b/server/workers/workers_8/worker.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import gzip +import redis + +import shutil +import datetime +import configparser + +def data_incorrect_format(session_uuid): + print('Incorrect format') + sys.exit(1) + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) + +host_redis_metadata = "localhost" +port_redis_metadata = 6380 + +redis_server_metadata = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=0) + +redis_server_analyzer = redis.StrictRedis( + host=host_redis_metadata, + port=port_redis_metadata, + db=2) + +# get file config +config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') +config_server = configparser.ConfigParser() +config_server.read(config_file_server) + +# get data directory +use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +# check if field is None +if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') +else: + data_directory = config_server['Save_Directories'].get('save_directory') + + +type = 8 +rotation_save_cycle = 300 #seconds + +analyzer_list_max_default_size = 10000 + +max_buffer_length = 10000 + +save_to_file = True + +def compress_file(file_full_path, i=0): + if i==0: + compressed_filename = '{}.gz'.format(file_full_path) + else: + compressed_filename = '{}.{}.gz'.format(file_full_path, i) + if os.path.isfile(compressed_filename): + compress_file(file_full_path, i+1) + else: + with open(file_full_path, 'rb') as f_in: + with gzip.open(compressed_filename, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + os.remove(file_full_path) + +def get_save_dir(dir_data_uuid, year, month, day): + dir_path = os.path.join(dir_data_uuid, year, month, day) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + return dir_path + +if __name__ == "__main__": + + if len(sys.argv) != 2: + print('usage:', 'Worker.py', 'session_uuid') + exit(1) + + session_uuid = sys.argv[1] + stream_name = 'stream:{}:{}'.format(type, session_uuid) + id = '0' + buffer = b'' + + # track launched worker + redis_server_stream.sadd('working_session_uuid:{}'.format(type), session_uuid) + + # get uuid + res = redis_server_stream.xread({stream_name: id}, count=1) + if res: + uuid = res[0][1][0][1][b'uuid'].decode() + # init file rotation + if save_to_file: + rotate_file = False + time_file = time.time() + date_file = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + dir_data_uuid = os.path.join(data_directory, uuid, str(type)) + dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8]) + filename = '{}-{}-{}-{}-{}.passivedns.txt'.format(uuid, date_file[0:4], date_file[4:6], date_file[6:8], date_file[8:14]) + save_path = os.path.join(dir_full_path, filename) + + print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) + else: + ########################### # TODO: clean db on error + print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid)) + sys.exit(1) + + while True: + + res = redis_server_stream.xread({stream_name: id}, count=1) + if res: + new_id = res[0][1][0][0].decode() + if id != new_id: + id = new_id + data = res[0][1][0][1] + + if id and data: + # reconstruct data + if buffer != b'': + data[b'message'] = b''.join([buffer, data[b'message']]) + buffer = b'' + + # send data to redis + # new line in received data + if b'\n' in data[b'message']: + all_line = data[b'message'].split(b'\n') + for line in all_line[:-1]: + for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)): + analyzer_uuid = analyzer_uuid.decode() + redis_server_analyzer.lpush('analyzer:{}:{}'.format(type, analyzer_uuid), line) + redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time()) + analyser_queue_max_size = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') + if analyser_queue_max_size is None: + analyser_queue_max_size = analyzer_list_max_default_size + redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) + # keep incomplete line + if all_line[-1] != b'': + buffer += all_line[-1] + else: + if len(buffer) < max_buffer_length: + buffer += data[b'message'] + else: + print('Error, infinite loop, max buffer length reached') + # force new line + buffer += b''.join([ data[b'message'], b'\n' ]) + + + # save data on disk + if save_to_file: + new_date = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + # check if a new rotation is needed + if ( new_date[0:8] != date_file[0:8] ) or ( time.time() - time_file > rotation_save_cycle ): + date_file = new_date + rotate_file = True + + # file rotation + if rotate_file and b'\n' in data[b'message']: + end_file, start_new_file = data[b'message'].rsplit(b'\n', maxsplit=1) + # save end of file + with open(save_path, 'ab') as f: + f.write(end_file) + compress_file(save_path) + + # get new save_path + dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8]) + filename = '{}-{}-{}-{}-{}.passivedns.txt'.format(uuid, date_file[0:4], date_file[4:6], date_file[6:8], date_file[8:14]) + save_path = os.path.join(dir_full_path, filename) + + # save start of new file + if start_new_file != b'': + with open(save_path, 'ab') as f: + f.write(start_new_file) + # end of rotation + rotate_file = False + time_file = time.time() + + else: + with open(save_path, 'ab') as f: + f.write(data[b'message']) + + redis_server_stream.xdel(stream_name, id) + + else: + # sucess, all data are saved + if redis_server_stream.sismember('ended_session', session_uuid): + redis_server_stream.srem('ended_session', session_uuid) + redis_server_stream.srem('session_uuid:{}'.format(type), session_uuid) + redis_server_stream.srem('working_session_uuid:{}'.format(type), session_uuid) + redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid) + redis_server_stream.delete(stream_name) + try: + if os.path.isfile(save_path): + print('save') + compress_file(save_path) + except NameError: + pass + print('---- passivedns DONE, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) + sys.exit(0) + else: + time.sleep(10) diff --git a/server/workers/workers_8/workers_manager.py b/server/workers/workers_8/workers_manager.py new file mode 100755 index 0000000..c1e64d9 --- /dev/null +++ b/server/workers/workers_8/workers_manager.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import redis +import subprocess + +host_redis_stream = "localhost" +port_redis_stream = 6379 + +redis_server_stream = redis.StrictRedis( + host=host_redis_stream, + port=port_redis_stream, + db=0) +type = 8 + +try: + redis_server_stream.ping() +except redis.exceptions.ConnectionError: + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + sys.exit(1) + +if __name__ == "__main__": + stream_name = 'stream:{}'.format(type) + redis_server_stream.delete('working_session_uuid:{}'.format(type)) + + while True: + for session_uuid in redis_server_stream.smembers('session_uuid:{}'.format(type)): + session_uuid = session_uuid.decode() + if not redis_server_stream.sismember('working_session_uuid:{}'.format(type), session_uuid): + + process = subprocess.Popen(['./worker.py', session_uuid]) + print('Launching new worker{} ... session_uuid={}'.format(type, session_uuid)) + + #print('.') + time.sleep(10)