From 44b150ff3edb1d381901acb9f0c089194363fc2a Mon Sep 17 00:00:00 2001 From: Sebastien Tricaud Date: Sun, 3 Nov 2019 22:11:56 -0800 Subject: [PATCH 1/2] Adding first skeleton for the SightingDB format --- sightingdb-format/Makefile | 8 +++++ sightingdb-format/raw.md | 61 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 sightingdb-format/Makefile create mode 100755 sightingdb-format/raw.md diff --git a/sightingdb-format/Makefile b/sightingdb-format/Makefile new file mode 100644 index 0000000..73b05d7 --- /dev/null +++ b/sightingdb-format/Makefile @@ -0,0 +1,8 @@ +MMARK:=mmark -xml2 -page + +docs = $(wildcard *.md) + +all: $(docs) + $(MMARK) $< > $<.xml + xml2rfc --text $<.xml + xml2rfc --html $<.xml diff --git a/sightingdb-format/raw.md b/sightingdb-format/raw.md new file mode 100755 index 0000000..06823e1 --- /dev/null +++ b/sightingdb-format/raw.md @@ -0,0 +1,61 @@ +%%% +Title = "SightingDB format" +abbrev = "SightingDB format" +category = "info" +docName = "draft-tricaud-sightingdb-format" +ipr= "trust200902" +area = "Security" + +date = 2019-03-03T00:00:00Z + +[[author]] +initials="S." +surname="Tricaud" +fullname="Sebastien Tricaud" +abbrev="Devo Inc." +organization = "Devo Inc." + [author.address] + email = "sebastien.tricaud@devo.com" + phone = "+1 866-221-2254" + [author.address.postal] + street = "150 Cambridgepark Drive" + city = "Cambridge, MA" + code = "02140" + country = "USA" +%%% + +.# Abstract + +This document describes the format used by SightingDB to give automated context to a given Attribute +by counting occurences and tracking times of observability. +SightingDB was designed to provide to MISP a Scalable and Fast way to store and retrive Attributes. + +{mainmatter} + +# Introduction + +Adding context to any Attribute is the key that makes it useful. While there exist numerous ways of doing it, +SightingDB does it by just counting. +Whenever somebody retrieves an Attribute, this counting is provided, allowing anyone to understand wether something +was observed few or many times. + +## Conventions and Terminology + +The key words "**MUST**", "**MUST NOT**", "**REQUIRED**", "**SHALL**", "**SHALL NOT**", +"**SHOULD**", "**SHOULD NOT**", "**RECOMMENDED**", "**MAY**", and "**OPTIONAL**" in this +document are to be interpreted as described in RFC 2119 [@!RFC2119]. + +# Format + +## Overview + +The SightingDB format is in the JSON [@!RFC8259] format. In SightingDB, a Sighting Object is composed of a single JSON object. + +# Acknowledgements + +The author wish to thank all the MISP community who are supporting the creation +of open standards in threat intelligence sharing. As well as amazing feedback gathered +during the MISP Summit 2019 in Luxembourg, in particular with Alexandre Dulaunoy and +Andras Iklody. + +{backmatter} From ac9006e9ef31771dfda798dabf956fc292512f87 Mon Sep 17 00:00:00 2001 From: Sebastien Tricaud Date: Mon, 4 Nov 2019 22:26:23 -0800 Subject: [PATCH 2/2] Added description for each field used by SightingDB --- sightingdb-format/raw.md | 81 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/sightingdb-format/raw.md b/sightingdb-format/raw.md index 06823e1..a4ec0d2 100755 --- a/sightingdb-format/raw.md +++ b/sightingdb-format/raw.md @@ -6,7 +6,7 @@ docName = "draft-tricaud-sightingdb-format" ipr= "trust200902" area = "Security" -date = 2019-03-03T00:00:00Z +date = 2019-11-03T00:00:00Z [[author]] initials="S." @@ -49,7 +49,84 @@ document are to be interpreted as described in RFC 2119 [@!RFC2119]. ## Overview -The SightingDB format is in the JSON [@!RFC8259] format. In SightingDB, a Sighting Object is composed of a single JSON object. +The SightingDB format is in the JSON [@!RFC8259] format. In SightingDB, a Sighting Object is composed of a single JSON object. This object contains the following fields: value, first_seen, last_seen, count, tags, ttl, frequency and manifold. + +### Attribute Storage + +The fields described previously describe an Attribute and all the required characteristics. However they are stored in a Namespace. A Namespace is similar to a path in a filesystem where the same file can be stored in multiple places. + +### Namespace + +A Namespace with multiple levels MUST be separated with the slash '/' character. There is no specification on how they are structured, since it depends on the use cases. + +A Namespace starting with the underscore '_' character means it is private and internal to SightingDB. There are all reserved for the engine and MUST NOT be used. + +Reserved namespaces are: +_expired/: Which contains all the attributes that expired, preserving the origin namespace +_shadow/: When a value is searched and does not exists, it is stored there +_stats: Statistics +_config: Configuration +_all: All the Attributes in one place, used to retrieve the 'manifold' property. + +The Attribute Key MUST always be the last part of the Namespace. + +#### Sample Namespaces + +/Organization1/service/ipv4: Store values for ipv4 keys in /Organization1/service +/everything/domain: Store domains in /everything + +### Attribute fields + +#### value + +The attribute value, used to store and retrieve information about an attribute. Note that value is not returned back in the JSON object, since it is queried, it is known. + +#### first_seen + +Time in UTC of the first time this value was captured + +#### last_seen + +Time in UTC of the last time this value was captured + +#### count + +How many time this value was written + +#### tags + +Tags follow how they are defined in MISP using the MISP Taxonomy. Each Tag is separated with the ';' character. + +#### ttl + +Time To Live, represents the expiration in seconds since the time the Attribute was created. Once it has expired, it moves in the private Namespace _expired. + +When an Attribute has this field set to 0, it means it is not set to expired. This is the default behavior. + +When an Attribute has this field set to a number greater than 0, the expiration status is computed only at retrieval time. + +#### frequency + +Frequency is the number of time an Attribute is seen in average per day. As this field can introduced latence, its implementation is OPTIONAL. + +#### manifold + +When a given Attribute Value is stored in different namespaces, the manifold field keeps track of them so it returns in how many different places this attributes exists. This is a simple counter. + +## SightingDB Format - One Attribute + +~~~~ +{ + "value":"127.0.0.1", + "first_seen":1530394819, + "last_seen":1572933618, + "count":578391, + "tags":"", + "ttl":0, + "frequency":1185, + "manifold": 17 +} +~~~~ # Acknowledgements