From 0ef9060ed09288950843fe899ea59bc0558295fe Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Mon, 27 Aug 2018 16:27:56 -0400 Subject: [PATCH] added 'to_pattern()' utility for pattern expressions; more information on patterning guide --- docs/guide/patterns.ipynb | 332 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 docs/guide/patterns.ipynb diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb new file mode 100644 index 0000000..0f79f7f --- /dev/null +++ b/docs/guide/patterns.ipynb @@ -0,0 +1,332 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# STIX Patterns\n", + "\n", + "python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher](https://github.com/oasis-open/cti-pattern-matcher)\n", + "\n", + "## Intro\n", + "\n", + "python-stix2 patterns are built compositely from the bottom up, creating components at sublevels before those at higher levels.\n", + "\n", + "## Examples\n", + "\n", + "### Comparison Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Equality Comparison Expressions:\n", + "\n", + "\t[domain-name:value = 'site.of.interest.zaz']\n", + "\n", + "\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n", + "\n", + "\n", + "Greater-than Comparison Expression:\n", + "\n", + "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", + "\n", + "\n", + "Is-Subset Comparison Expression:\n", + "\n", + "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import DomainName, File, IPv4Address\n", + "from stix2 import (ObjectPath, EqualityComparisonExpression, GreaterThanComparisonExpression,\n", + " IsSubsetComparisonExpression, FloatConstant, StringConstant)\n", + "\n", + "# ---- Equality Comparison expressions\n", + "print(\"Equality Comparison Expressions:\\n\")\n", + "\n", + "lhs = ObjectPath(\"domain-name\", [\"value\"])\n", + "ece_1 = EqualityComparisonExpression(lhs, \"site.of.interest.zaz\")\n", + "print(\"\\t{}\\n\".format(ece_1.to_pattern()))\n", + "\n", + "lhs = ObjectPath(\"file\", [\"parent_directory_ref\",\"path\"])\n", + "ece_2 = EqualityComparisonExpression(lhs, \"C:\\\\Windows\\\\System32\")\n", + "print(\"\\t{}\\n\".format(ece_2.to_pattern()))\n", + "\n", + "# Greater-than Comparison expressions\n", + "print(\"\\nGreater-than Comparison Expression:\\n\")\n", + "\n", + "lhs = ObjectPath(\"file\", [\"extensions\", \"windows-pebinary-ext\", \"sections[*]\", \"entropy\"])\n", + "gte = GreaterThanComparisonExpression(lhs, FloatConstant(\"7.0\"))\n", + "print(\"\\t{}\\n\".format(gte.to_pattern()))\n", + "\n", + "# IsSubset Comparison expressions\n", + "print(\"\\nIs-Subset Comparison Expression:\\n\")\n", + "\n", + "lhs = ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"])\n", + "iss = IsSubsetComparisonExpression(lhs, StringConstant(\"2001:0db8:dead:beef:0000:0000:0000:0000/64\"))\n", + "print(\"\\t{}\\n\".format(iss.to_pattern()))\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Observation Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Observation Expressions:\n", + "\n", + "(AND)\n", + "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n", + "\n", + "(OR)\n", + "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", + "\n", + "(OR,AND)\n", + "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", + "\n", + "(AND,OR,OR)\n", + "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", + "\n", + "(FollowedBy)\n", + "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import (IntegerConstant, HashConstant, ObjectPath,\n", + " EqualityComparisonExpression, AndBooleanExpression,\n", + " OrBooleanExpression, ParentheticalExpression,\n", + " AndObservationExpression, OrObservationExpression,\n", + " FollowedByObservationExpression)\n", + "\n", + "# ---- Observation expressions\n", + "print(\"Observation Expressions:\\n\")\n", + "\n", + "# AND boolean\n", + "ece3 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"sender_ref\", \"value\"]), \"stark@example.com\")\n", + "ece4 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"subject\"]), \"Conference Info\")\n", + "abe = AndBooleanExpression([ece3, ece4])\n", + "print(\"(AND)\\n{}\\n\".format(abe.to_pattern()))\n", + "\n", + "# OR boolean\n", + "ece5 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/foo\")\n", + "ece6 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/bar\")\n", + "obe = OrBooleanExpression([ece5, ece6])\n", + "print(\"(OR)\\n{}\\n\".format(obe.to_pattern()))\n", + "\n", + "# ( OR ) AND boolean\n", + "ece7 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"pdf.exe\")\n", + "ece8 = EqualityComparisonExpression(ObjectPath(\"file\", [\"size\"]), IntegerConstant(\"371712\"))\n", + "ece9 = EqualityComparisonExpression(ObjectPath(\"file\", [\"created\"]), \"2014-01-13T07:03:17Z\")\n", + "obe1 = OrBooleanExpression([ece7, ece8])\n", + "pobe = ParentheticalExpression(obe1)\n", + "abe1 = AndBooleanExpression([pobe, ece9])\n", + "print(\"(OR,AND)\\n{}\\n\".format(abe1.to_pattern()))\n", + "\n", + "# ( AND ) OR ( OR ) observation\n", + "ece20 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\")\n", + "ece21 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", + "ece22 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"fooproc\")\n", + "ece23 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"procfoo\")\n", + "# NOTE: we need to use AND/OR observation expression instead of just boolean \n", + "# expressions as the operands are not on the same object-type\n", + "aoe = ParentheticalExpression(AndObservationExpression([ece20, ece21]))\n", + "obe2 = OrBooleanExpression([ece22, ece23])\n", + "ooe = OrObservationExpression([aoe, obe2])\n", + "print(\"(AND,OR,OR)\\n{}\\n\".format(ooe.to_pattern()))\n", + "\n", + "# FOLLOWED-BY\n", + "ece10 = EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\"))\n", + "ece11 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "print(\"(FollowedBy)\\n{}\\n\".format(fbe.to_pattern()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__See [note](#BooleanExpressions-vs-CompoundObservationExpressions) on when to use BooleanExpressions vs CompoundObservationExpressions__" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Qualified Observation Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Qualified Observation Expressions:\n", + "\n", + "(WITHIN)\n", + "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar'] WITHIN 300 SECONDS\n", + "\n", + "(REPEAT, WITHIN)\n", + "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n", + "\n", + "(START-STOP)\n", + "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import (TimestampConstant, HashConstant, ObjectPath, EqualityComparisonExpression,\n", + " AndBooleanExpression, WithinQualifier, RepeatQualifier, StartStopQualifier,\n", + " QualifiedObservationExpression, FollowedByObservationExpression)\n", + "\n", + "# Qualified Observation Expressions\n", + "print(\"Qualified Observation Expressions:\\n\")\n", + "\n", + "# WITHIN\n", + "ece10 = EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\"))\n", + "ece11 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "qoe = QualifiedObservationExpression(fbe, WithinQualifier(300))\n", + "print(\"(WITHIN)\\n{}\\n\".format(qoe.to_pattern()))\n", + "\n", + "# REPEATS, WITHIN\n", + "ece12 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"type\"]), \"domain-name\")\n", + "ece13 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"]), \"example.com\")\n", + "abe2 = AndBooleanExpression([ece12, ece13])\n", + "qoe1 = QualifiedObservationExpression(QualifiedObservationExpression(abe2, RepeatQualifier(5)), WithinQualifier(180))\n", + "print(\"(REPEAT, WITHIN)\\n{}\\n\".format(qoe1.to_pattern()))\n", + "\n", + "# START, STOP\n", + "ece14 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\")\n", + "ssq = StartStopQualifier(TimestampConstant('2016-06-01T00:00:00Z'), TimestampConstant('2016-07-01T00:00:00Z'))\n", + "qoe2 = QualifiedObservationExpression(ece14, ssq)\n", + "print(\"(START-STOP)\\n{}\\n\".format(qoe2.to_pattern()))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Attaching patterns to STIX2 Domain objects\n", + "\n", + "As seen in the previous examples, the *__to_pattern()__* utility is used to display the pattern. However, the true purpose of this utility is to be able to seamlessly add a constructed pattern to an python-stix2 object.\n", + "\n", + "The *__to_pattern()__* utility constructs a string version of the pattern that adheres to the STIX2 patterning language specification.\n", + "\n", + "### Example" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--9f2d8014-3c2f-4127-bc2c-a209913404a5\",\n", + " \"created\": \"2018-08-27T18:57:00.762Z\",\n", + " \"modified\": \"2018-08-27T18:57:00.762Z\",\n", + " \"name\": \"Cryptotorch\",\n", + " \"pattern\": \"[file:name = '$$t00rzch$$.elf']\",\n", + " \"valid_from\": \"2018-08-27T18:57:00.762972Z\",\n", + " \"labels\": [\n", + " \"malware\",\n", + " \"ransomware\"\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "from stix2 import Indicator, EqualityComparisonExpression\n", + "\n", + "ece14 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"$$t00rzch$$.elf\")\n", + "ind = Indicator(name=\"Cryptotorch\", labels=[\"malware\", \"ransomware\"], pattern=\"[{}]\".format(ece14))\n", + "print(ind)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BooleanExpressions vs CompoundObservationExpressions\n", + "\n", + "Be careful to note the difference between these two very similar pattern components. \n", + "\n", + "__BooleanExpressions__\n", + " - stix2.AndBooleanExpression\n", + " - stix2.booleanExpression\n", + " \n", + " __Usage__: When the boolean sub-expressions refer to the same root object \n", + "\n", + " __Example__:\n", + " ```[domain-name:value = \"www.5z8.info\" AND domain-name:resolvess_to_refs[*].value = \"'198.51.100.1/32'\"]```\n", + " \n", + " __Rendering__: when pattern is rendered, brackets or parenthesis will encapsulate boolean expression\n", + " \n", + "__CompoundObservationExpressions__\n", + " - stix2.AndObservationExpression\n", + " - stix2.OrObservationExpression\n", + " \n", + " __Usage__: When the boolean sub-expressions refer to different root objects\n", + "\n", + " __Example__:\n", + " ```[file:name=\"foo.dll\"] AND [process:name = \"procfoo\"]```\n", + " \n", + " __Rendering__: when pattern is rendered, brackets will encapsulate each boolean sub-expression\n", + " \n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}