MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #131 from oasis-open/taxii_collection_source_404 

Data source/store/sink changes
stix2.0
Chris Lenk 2018-02-27 17:04:14 -05:00 committed by GitHub
parent 554cd081b2 b6f14cdd4d
commit 9402cff110
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 969 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

294
docs/guide/datastore.ipynb
View File

@ -52,6 +52,19 @@
"print = json_print"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# without this configuration, only last print() call is outputted in cells\n",
"from IPython.core.interactiveshell import InteractiveShell\n",
"InteractiveShell.ast_node_interactivity = \"all\""
]
},
{
"cell_type": "markdown",
"metadata": {},
@ -85,50 +98,229 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"type\": \"indicator\",\n",
" \"id\": \"indicator--797ae2b5-3f7a-44c5-8ecd-33ba22fdc2b5\",\n",
" \"created\": \"2017-10-04T19:27:41.000Z\",\n",
" \"modified\": \"2017-10-04T19:27:41.000Z\",\n",
" \"labels\": [\n",
" \"malicious-activity\"\n",
" ],\n",
" \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n",
" \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n",
" \"valid_from\": \"2017-10-04T19:27:41Z\",\n",
" \"kill_chain_phases\": [\n",
" {\n",
" \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n",
" \"phase_name\": \"delivery\"\n",
" }\n",
" ]\n",
"}\n",
"{\n",
" \"type\": \"indicator\",\n",
" \"id\": \"indicator--11913f42-2d52-4b9d-842f-94bf06819a66\",\n",
" \"created\": \"2017-10-04T19:27:41.000Z\",\n",
" \"modified\": \"2017-10-04T19:27:41.000Z\",\n",
" \"labels\": [\n",
" \"malicious-activity\"\n",
" ],\n",
" \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n",
" \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n",
" \"valid_from\": \"2017-10-04T19:27:41Z\",\n",
" \"kill_chain_phases\": [\n",
" {\n",
" \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n",
" \"phase_name\": \"delivery\"\n",
" }\n",
" ]\n",
"}\n"
]
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;intrusion-set&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created_by_ref&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-31T21:31:53.197Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-31T21:31:53.197Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;DragonOK&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;aliases&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;DragonOK&quot;</span>\n",
" <span class=\"p\">],</span>\n",
" <span class=\"nt\">&quot;external_references&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;source_name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;mitre-attack&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;url&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;https://attack.mitre.org/wiki/Group/G0017&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;external_id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;G0017&quot;</span>\n",
" <span class=\"p\">},</span>\n",
" <span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;source_name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Operation Quantum Entanglement&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;url&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/wp-operation-quantum-entanglement.pdf&quot;</span>\n",
" <span class=\"p\">},</span>\n",
" <span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;source_name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Symbiotic APT Groups&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;url&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;https://dl.mandiant.com/EE/library/MIRcon2014/MIRcon%202014%20R&amp;D%20Track%20Insight%20into%20Symbiotic%20APT.pdf&quot;</span>\n",
" <span class=\"p\">},</span>\n",
" <span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;source_name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;New DragonOK&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;url&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;http://researchcenter.paloaltonetworks.com/2015/04/unit-42-identifies-new-dragonok-backdoor-malware-deployed-against-japanese-targets/&quot;</span>\n",
" <span class=\"p\">}</span>\n",
" <span class=\"p\">],</span>\n",
" <span class=\"nt\">&quot;object_marking_refs&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168&quot;</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;indicator&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;indicator--02b90f02-a96a-43ee-88f1-1e87297941f2&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-13T07:00:24.000Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-13T07:00:24.000Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Ransomware IP Blocklist&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;IP Blocklist address from abuse.ch&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;pattern&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;[ ipv4-addr:value = &#39;91.237.247.24&#39; ]&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;valid_from&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-13T07:00:24Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;labels&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;malicious-activity&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"s2\">&quot;Ransomware&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"s2\">&quot;Botnet&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"s2\">&quot;C&amp;C&quot;</span>\n",
" <span class=\"p\">],</span>\n",
" <span class=\"nt\">&quot;external_references&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;source_name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;abuse.ch&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;url&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;https://ransomwaretracker.abuse.ch/blocklist/&quot;</span>\n",
" <span class=\"p\">}</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
@ -136,22 +328,22 @@
"from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n",
"\n",
"# create FileSystemStore\n",
"fs = FileSystemSource(\"/tmp/stix2_data\")\n",
"fs = FileSystemSource(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/\")\n",
"\n",
"# create TAXIICollectionSource\n",
"colxn = Collection('https://test.freetaxii.com:8000/api1/collections/9cfa669c-ee94-4ece-afd2-f8edac37d8fd/')\n",
"colxn = Collection('https://test.freetaxii.com:8000/osint/collections/a9c22eaf-0f3e-482c-8bb4-45ae09e75d9b/')\n",
"ts = TAXIICollectionSource(colxn)\n",
"\n",
"# add them both to the CompositeDataSource\n",
"cs = CompositeDataSource()\n",
"cs.add_data_sources([fs, ts])\n",
"cs.add_data_sources([fs,ts])\n",
"\n",
"# get an object that is only in the filesystem\n",
"ta = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n",
"print(ta)\n",
"intrusion_set = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n",
"print(intrusion_set)\n",
"\n",
"# get an object that is only in the TAXII collection\n",
"ind = cs.get('indicator--37a6a5de-a5b9-425a-903a-4ae9cbf1ff3f')\n",
"ind = cs.get('indicator--02b90f02-a96a-43ee-88f1-1e87297941f2')\n",
"print(ind)\n"
]
},
@ -197,7 +389,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {
"collapsed": true
},
@ -231,7 +423,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {
"collapsed": true
},
@ -537,9 +729,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "cti-python-stix2",
"language": "python",
"name": "python2"
"name": "cti-python-stix2"
},
"language_info": {
"codemirror_mode": {

8
docs/guide/filesystem.ipynb
View File

@ -82,7 +82,7 @@
" /STIX2 Domain Object type\n",
"```\n",
"\n",
"Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n",
"The master STIX2 content directory contains subdirectories, each of which aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\", etc.). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n",
"\n",
"```\n",
"stix2_content/\n",
@ -109,11 +109,11 @@
"\n",
"[FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) that point the same file directory.\n",
"\n",
"Use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n",
"For use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX2 content will be retrieved from one distinct file directory and pushed to another.\n",
"\n",
"### FileSystem API\n",
"\n",
"A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query). The format of the STIX2 content targeted by the FileSystem suite is JSON files. When STIX2 content (in JSON) is retrieved by the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n",
"A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) retrieves STIX2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n",
"\n",
"A note on [add()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n",
"\n",
@ -264,7 +264,7 @@
"source": [
"from stix2 import FileSystemSource\n",
"\"\"\"\n",
"Working with FileSystemSource for retrieveing STIX content.\n",
"Working with FileSystemSource for retrieving STIX content.\n",
"\"\"\"\n",
"\n",
"# create FileSystemSource\n",

582
docs/guide/memory.ipynb
View File

@ -62,10 +62,11 @@
"\n",
"\n",
"### Memory API\n",
"A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects, while (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n",
"\n",
"A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) and [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). These methods both add STIX content to an internal dictionary (maintained by [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore)). STIX content that is to be added can be in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as, is how it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dictionaries (of STIX objects) are stored as Python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). Also, for [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, individually or in a Bundle. \n",
"A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file): For [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n",
"\n",
"A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). This method dumps all STIX content that is in [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored (i.e. supplied) to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n",
"A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored in (i.e. supplied to) the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n",
"\n",
"### Memory Examples\n",
"\n",
@ -74,26 +75,101 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"type\": \"indicator\",\n",
" \"id\": \"indicator--d91ef175-8a82-470a-a610-bbd2ee8a1516\",\n",
" \"created\": \"2017-09-29T19:52:16.930Z\",\n",
" \"modified\": \"2017-09-29T19:52:16.930Z\",\n",
" \"labels\": [\n",
" \"malicious-activity\"\n",
" ],\n",
" \"description\": \"Crusades C2 implant\",\n",
" \"pattern\": \"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\",\n",
" \"valid_from\": \"2017-09-29T19:52:16.930909Z\"\n",
"}\n"
]
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;indicator&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;indicator--2f61e4e7-0891-4e09-b79a-66f5e594fec0&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.590Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.590Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Crusades C2 implant&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;pattern&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;[file:hashes.&#39;SHA-256&#39; = &#39;54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b&#39;]&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;valid_from&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.590939Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;labels&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;malicious-activity&quot;</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
@ -115,26 +191,101 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"type\": \"indicator\",\n",
" \"id\": \"indicator--79fdaad7-c461-49bb-ad1d-caa5e9c51c90\",\n",
" \"created\": \"2017-09-29T19:52:17.021Z\",\n",
" \"modified\": \"2017-09-29T19:52:17.021Z\",\n",
" \"labels\": [\n",
" \"malicious-activity\"\n",
" ],\n",
" \"description\": \"Crusades stage 2 implant variant\",\n",
" \"pattern\": \"[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']\",\n",
" \"valid_from\": \"2017-09-29T19:52:17.021728Z\"\n",
"}\n"
]
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;indicator&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;indicator--ddb765ba-ff1e-4285-bf33-1f6d08f583d6&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.799Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.799Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;description&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Crusades stage 2 implant variant&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;pattern&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;[file:hashes.&#39;SHA-256&#39; = &#39;31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c&#39;]&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;valid_from&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.799228Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;labels&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;malicious-activity&quot;</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
@ -157,89 +308,216 @@
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": 5,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"-----------------------\n",
"{'name': 'Urban2', 'created': '2017-09-12T13:26:18.023Z', 'labels': ['rootkit'], 'modified': '2017-09-12T13:26:18.023Z', 'type': 'malware', 'id': 'malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4'}\n",
"-----------------------\n",
"{\n",
" \"type\": \"malware\",\n",
" \"id\": \"malware--2b3dd412-18a5-4e81-8742-4977068eb3eb\",\n",
" \"created\": \"2017-09-29T19:52:17.028Z\",\n",
" \"modified\": \"2017-09-29T19:52:17.028Z\",\n",
" \"name\": \"Alexios\",\n",
" \"labels\": [\n",
" \"rootkit\"\n",
" ]\n",
"}\n"
]
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;malware&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;malware--e8170e70-522f-4ec3-aa22-afb55bfad0b0&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.806Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-11-17T17:01:31.806Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Alexios&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;labels&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;rootkit&quot;</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from stix2 import Filter\n",
"\n",
"# add dictionary (of STIX object) to MemoryStore\n",
"# (this dict would assumably come from output of another source,\n",
"# i.e. a loaded json file, NOT manually created as done here for sample purposes)\n",
"\n",
"malware = {\n",
" \"type\": \"malware\",\n",
" \"id\" : \"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\",\n",
" \"labels\": [\"rootkit\"],\n",
" \"name\": \"Urban2\",\n",
" \"created\": \"2017-09-12T13:26:18.023Z\",\n",
" \"modified\": \"2017-09-12T13:26:18.023Z\"\n",
"}\n",
"\n",
"mem.add(malware)\n",
"\n",
"results = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])\n",
"for r in results:\n",
" # note that python STIX objects are pretty-printed\n",
" # due to some python dunder method magic, but normal\n",
" # python dictionaries are not by default. Thus the\n",
" # python STIX objects and python STIX dictionaries\n",
" # that match the above query can be easily identified visually\n",
" print(\"-----------------------\")\n",
" print(r)"
"mal = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])[0]\n",
"print(mal)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"type\": \"report\",\n",
" \"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\n",
" \"created\": \"2017-05-08T18:34:08.042Z\",\n",
" \"modified\": \"2017-05-08T18:34:08.042Z\",\n",
" \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\",\n",
" \"published\": \"2017-05-08T10:24:11.011Z\",\n",
" \"object_refs\": [\n",
" \"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"\n",
" ],\n",
" \"labels\": [\n",
" \"threat-report\"\n",
" ]\n",
"}\n"
]
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;report&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;report--2add14d6-bbf3-4308-bb8e-226d314a08e4&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-08T18:34:08.042Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-08T18:34:08.042Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;published&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-08T10:24:11.011Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;object_refs&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4&quot;</span>\n",
" <span class=\"p\">],</span>\n",
" <span class=\"nt\">&quot;labels&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;threat-report&quot;</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from stix2 import Filter\n",
"\n",
"# add json formatted string to MemoryStore\n",
"# Again, would NOT manual create json-formatted string\n",
"# Again, would NOT manually create json-formatted string\n",
"# but taken as an output form from another source\n",
"report = '{\"type\": \"report\",\"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\"labels\": [\"threat-report\"], \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\", \"published\": \"2017-05-08T10:24:11.011Z\", \"object_refs\":[\"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"], \"created\": \"2017-05-08T18:34:08.042Z\", \"modified\": \"2017-05-08T18:34:08.042Z\"}'\n",
"\n",
@ -257,15 +535,103 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{u'name': u'The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.', u'created': u'2017-05-08T18:34:08.042Z', u'labels': [u'threat-report'], u'modified': u'2017-05-08T18:34:08.042Z', u'object_refs': [u'malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4'], u'published': u'2017-05-08T10:24:11.011Z', u'type': u'report', u'id': u'report--2add14d6-bbf3-4308-bb8e-226d314a08e4'}\n"
]
"data": {
"text/html": [
"<style type=\"text/css\">.highlight .hll { background-color: #ffffcc }\n",
".highlight { background: #f8f8f8; }\n",
".highlight .c { color: #408080; font-style: italic } /* Comment */\n",
".highlight .err { border: 1px solid #FF0000 } /* Error */\n",
".highlight .k { color: #008000; font-weight: bold } /* Keyword */\n",
".highlight .o { color: #666666 } /* Operator */\n",
".highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
".highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
".highlight .cp { color: #BC7A00 } /* Comment.Preproc */\n",
".highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
".highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
".highlight .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
".highlight .gd { color: #A00000 } /* Generic.Deleted */\n",
".highlight .ge { font-style: italic } /* Generic.Emph */\n",
".highlight .gr { color: #FF0000 } /* Generic.Error */\n",
".highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
".highlight .gi { color: #00A000 } /* Generic.Inserted */\n",
".highlight .go { color: #888888 } /* Generic.Output */\n",
".highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
".highlight .gs { font-weight: bold } /* Generic.Strong */\n",
".highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
".highlight .gt { color: #0044DD } /* Generic.Traceback */\n",
".highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
".highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
".highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
".highlight .kp { color: #008000 } /* Keyword.Pseudo */\n",
".highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
".highlight .kt { color: #B00040 } /* Keyword.Type */\n",
".highlight .m { color: #666666 } /* Literal.Number */\n",
".highlight .s { color: #BA2121 } /* Literal.String */\n",
".highlight .na { color: #7D9029 } /* Name.Attribute */\n",
".highlight .nb { color: #008000 } /* Name.Builtin */\n",
".highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
".highlight .no { color: #880000 } /* Name.Constant */\n",
".highlight .nd { color: #AA22FF } /* Name.Decorator */\n",
".highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
".highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
".highlight .nf { color: #0000FF } /* Name.Function */\n",
".highlight .nl { color: #A0A000 } /* Name.Label */\n",
".highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
".highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
".highlight .nv { color: #19177C } /* Name.Variable */\n",
".highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
".highlight .w { color: #bbbbbb } /* Text.Whitespace */\n",
".highlight .mb { color: #666666 } /* Literal.Number.Bin */\n",
".highlight .mf { color: #666666 } /* Literal.Number.Float */\n",
".highlight .mh { color: #666666 } /* Literal.Number.Hex */\n",
".highlight .mi { color: #666666 } /* Literal.Number.Integer */\n",
".highlight .mo { color: #666666 } /* Literal.Number.Oct */\n",
".highlight .sa { color: #BA2121 } /* Literal.String.Affix */\n",
".highlight .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
".highlight .sc { color: #BA2121 } /* Literal.String.Char */\n",
".highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
".highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
".highlight .s2 { color: #BA2121 } /* Literal.String.Double */\n",
".highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
".highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
".highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
".highlight .sx { color: #008000 } /* Literal.String.Other */\n",
".highlight .sr { color: #BB6688 } /* Literal.String.Regex */\n",
".highlight .s1 { color: #BA2121 } /* Literal.String.Single */\n",
".highlight .ss { color: #19177C } /* Literal.String.Symbol */\n",
".highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
".highlight .fm { color: #0000FF } /* Name.Function.Magic */\n",
".highlight .vc { color: #19177C } /* Name.Variable.Class */\n",
".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
" <span class=\"nt\">&quot;type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;report&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;id&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;report--2add14d6-bbf3-4308-bb8e-226d314a08e4&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;created&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-08T18:34:08.042Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;modified&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-08T18:34:08.042Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;name&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;published&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;2017-05-08T10:24:11.011Z&quot;</span><span class=\"p\">,</span>\n",
" <span class=\"nt\">&quot;object_refs&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4&quot;</span>\n",
" <span class=\"p\">],</span>\n",
" <span class=\"nt\">&quot;labels&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
" <span class=\"s2\">&quot;threat-report&quot;</span>\n",
" <span class=\"p\">]</span>\n",
"<span class=\"p\">}</span>\n",
"</pre></div>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
@ -280,17 +646,15 @@
"report = mem_2.get(\"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\")\n",
"\n",
"# for visualpurposes\n",
"# Note: Since STIX content was added to MemoryStore as json,\n",
"# it is maintained as python dictionaries ( as opposed to STIX objects)\n",
"print(report)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "cti-python-stix2",
"language": "python",
"name": "python2"
"name": "cti-python-stix2"
},
"language_info": {
"codemirror_mode": {

4
docs/guide/taxii.ipynb
View File

@ -58,9 +58,9 @@
"source": [
"## TAXIICollection\n",
"\n",
"The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) for pushing and retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) for retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n",
"The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) pushes and retrieves STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) retrieves STIX content from local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) pushes STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n",
"\n",
"A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n",
"A note on TAXII2 searching/filtering of STIX content: TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n",
"\n",
"### TAXIICollection API\n",
"\n",

66
stix2/sources/filesystem.py
View File

@ -1,8 +1,6 @@
"""
Python STIX 2.0 FileSystem Source/Sink
TODO:
Test everything
"""
import json
@ -22,7 +20,12 @@ class FileSystemStore(DataStore):
Args:
stix_dir (str): path to directory of STIX objects
bundlify (bool): Whether to wrap objects in bundles when saving them.
allow_custom (bool): whether to allow custom STIX content to be
pushed/retrieved. Defaults to True for FileSystemSource side(retrieving data)
and False for FileSystemSink side(pushing data). However, when
parameter is supplied, it will be applied to both FileSystemSource
and FileSystemSink.
bundlify (bool): whether to wrap objects in bundles when saving them.
Default: False.
Attributes:
@ -30,10 +33,16 @@ class FileSystemStore(DataStore):
sink (FileSystemSink): FileSystemSink
"""
def __init__(self, stix_dir, bundlify=False):
def __init__(self, stix_dir, allow_custom=None, bundlify=False):
if allow_custom is None:
allow_custom_source = True
allow_custom_sink = False
else:
allow_custom_sink = allow_custom_source = allow_custom
super(FileSystemStore, self).__init__(
source=FileSystemSource(stix_dir=stix_dir),
sink=FileSystemSink(stix_dir=stix_dir, bundlify=bundlify)
source=FileSystemSource(stix_dir=stix_dir, allow_custom=allow_custom_source),
sink=FileSystemSink(stix_dir=stix_dir, allow_custom=allow_custom_sink, bundlify=bundlify)
)
@ -46,13 +55,16 @@ class FileSystemSink(DataSink):
Args:
stix_dir (str): path to directory of STIX objects.
allow_custom (bool): Whether to allow custom STIX content to be
added to the FileSystemSource. Default: False
bundlify (bool): Whether to wrap objects in bundles when saving them.
Default: False.
"""
def __init__(self, stix_dir, bundlify=False):
def __init__(self, stix_dir, allow_custom=False, bundlify=False):
super(FileSystemSink, self).__init__()
self._stix_dir = os.path.abspath(stix_dir)
self.allow_custom = allow_custom
self.bundlify = bundlify
if not os.path.exists(self._stix_dir):
@ -71,20 +83,18 @@ class FileSystemSink(DataSink):
os.makedirs(os.path.dirname(path))
if self.bundlify:
stix_obj = Bundle(stix_obj)
stix_obj = Bundle(stix_obj, allow_custom=self.allow_custom)
with open(path, "w") as f:
f.write(str(stix_obj))
def add(self, stix_data=None, allow_custom=False, version=None):
def add(self, stix_data=None, version=None):
"""Add STIX objects to file directory.
Args:
stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content
in a STIX object (or list of), dict (or list of), or a STIX 2.0
json encoded string.
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -100,24 +110,24 @@ class FileSystemSink(DataSink):
self._check_path_and_write(stix_data)
elif isinstance(stix_data, (str, dict)):
stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version)
if stix_data["type"] == "bundle":
# extract STIX objects
for stix_obj in stix_data.get("objects", []):
self.add(stix_obj, allow_custom=allow_custom, version=version)
self.add(stix_obj, version=version)
else:
# adding json-formatted STIX
self._check_path_and_write(stix_data)
self._check_path_and_write(stix_data,)
elif isinstance(stix_data, Bundle):
# recursively add individual STIX objects
for stix_obj in stix_data.get("objects", []):
self.add(stix_obj, allow_custom=allow_custom, version=version)
self.add(stix_obj, version=version)
elif isinstance(stix_data, list):
# recursively add individual STIX objects
for stix_obj in stix_data:
self.add(stix_obj, allow_custom=allow_custom, version=version)
self.add(stix_obj, version=version)
else:
raise TypeError("stix_data must be a STIX object (or list of), "
@ -134,11 +144,14 @@ class FileSystemSource(DataSource):
Args:
stix_dir (str): path to directory of STIX objects
allow_custom (bool): Whether to allow custom STIX content to be
added to the FileSystemSink. Default: True
"""
def __init__(self, stix_dir):
def __init__(self, stix_dir, allow_custom=True):
super(FileSystemSource, self).__init__()
self._stix_dir = os.path.abspath(stix_dir)
self.allow_custom = allow_custom
if not os.path.exists(self._stix_dir):
raise ValueError("directory path for STIX data does not exist: %s" % self._stix_dir)
@ -147,15 +160,13 @@ class FileSystemSource(DataSource):
def stix_dir(self):
return self._stix_dir
def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
def get(self, stix_id, version=None, _composite_filters=None):
"""Retrieve STIX object from file directory via STIX ID.
Args:
stix_id (str): The STIX ID of the STIX object to be retrieved.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -167,7 +178,7 @@ class FileSystemSource(DataSource):
"""
query = [Filter("id", "=", stix_id)]
all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
all_data = self.query(query=query, version=version, _composite_filters=_composite_filters)
if all_data:
stix_obj = sorted(all_data, key=lambda k: k['modified'])[0]
@ -176,7 +187,7 @@ class FileSystemSource(DataSource):
return stix_obj
def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
def all_versions(self, stix_id, version=None, _composite_filters=None):
"""Retrieve STIX object from file directory via STIX ID, all versions.
Note: Since FileSystem sources/sinks don't handle multiple versions
@ -186,8 +197,6 @@ class FileSystemSource(DataSource):
stix_id (str): The STIX ID of the STIX objects to be retrieved.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -197,9 +206,9 @@ class FileSystemSource(DataSource):
a python STIX objects and then returned
"""
return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)]
return [self.get(stix_id=stix_id, version=version, _composite_filters=_composite_filters)]
def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
def query(self, query=None, version=None, _composite_filters=None):
"""Search and retrieve STIX objects based on the complete query.
A "complete query" includes the filters from the query, the filters
@ -210,8 +219,6 @@ class FileSystemSource(DataSource):
query (list): list of filters to search on
_composite_filters (set): set of filters passed from the
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -221,6 +228,7 @@ class FileSystemSource(DataSource):
parsed into a python STIX objects and then returned.
"""
all_data = []
if query is None:
@ -304,7 +312,7 @@ class FileSystemSource(DataSource):
all_data = deduplicate(all_data)
# parse python STIX objects from the STIX object dicts
stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data]
stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data]
return stix_objs

87
stix2/sources/memory.py
View File

@ -1,9 +1,6 @@
"""
Python STIX 2.0 Memory Source/Sink
TODO:
Run through tests again, lot of changes.
TODO:
Use deduplicate() calls only when memory corpus is dirty (been added to)
can save a lot of time for successive queries
@ -24,7 +21,7 @@ from stix2.sources import DataSink, DataSource, DataStore
from stix2.sources.filters import Filter, apply_common_filters
def _add(store, stix_data=None, allow_custom=False, version=None):
def _add(store, stix_data=None, version=None):
"""Add STIX objects to MemoryStore/Sink.
Adds STIX objects to an in-memory dictionary for fast lookup.
@ -32,8 +29,6 @@ def _add(store, stix_data=None, allow_custom=False, version=None):
Args:
stix_data (list OR dict OR STIX object): STIX objects to be added
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -46,28 +41,19 @@ def _add(store, stix_data=None, allow_custom=False, version=None):
if stix_data["type"] == "bundle":
# adding a json bundle - so just grab STIX objects
for stix_obj in stix_data.get("objects", []):
_add(store, stix_obj, allow_custom=allow_custom, version=version)
_add(store, stix_obj, version=version)
else:
# adding a json STIX object
store._data[stix_data["id"]] = stix_data
elif isinstance(stix_data, str):
# adding json encoded string of STIX content
stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
if stix_data["type"] == "bundle":
# recurse on each STIX object in bundle
for stix_obj in stix_data.get("objects", []):
_add(store, stix_obj, allow_custom=allow_custom, version=version)
else:
_add(store, stix_data, allow_custom=allow_custom, version=version)
elif isinstance(stix_data, list):
# STIX objects are in a list- recurse on each object
for stix_obj in stix_data:
_add(store, stix_obj, allow_custom=allow_custom, version=version)
_add(store, stix_obj, version=version)
else:
raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle")
raise TypeError("stix_data expected to be a python-stix2 object (or list of), JSON formatted STIX (or list of),"
" or a JSON formatted STIX bundle. stix_data was of type: " + str(type(stix_data)))
class MemoryStore(DataStore):
@ -81,8 +67,9 @@ class MemoryStore(DataStore):
Args:
stix_data (list OR dict OR STIX object): STIX content to be added
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
allow_custom (bool): whether to allow custom STIX content.
Only applied when export/input functions called, i.e.
load_from_file() and save_to_file(). Defaults to True.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -92,11 +79,11 @@ class MemoryStore(DataStore):
sink (MemorySink): MemorySink
"""
def __init__(self, stix_data=None, allow_custom=False, version=None):
def __init__(self, stix_data=None, allow_custom=True, version=None):
self._data = {}
if stix_data:
_add(self, stix_data, allow_custom=allow_custom, version=version)
_add(self, stix_data, version=version)
super(MemoryStore, self).__init__(
source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True),
@ -109,8 +96,6 @@ class MemoryStore(DataStore):
Args:
file_path (str): file path to write STIX data to
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
"""
return self.sink.save_to_file(*args, **kwargs)
@ -123,8 +108,6 @@ class MemoryStore(DataStore):
Args:
file_path (str): file path to load STIX data from
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -141,37 +124,39 @@ class MemorySink(DataSink):
Args:
stix_data (dict OR list): valid STIX 2.0 content in
bundle or a list.
_store (bool): if the MemorySink is a part of a DataStore,
_store (bool): whether the MemorySink is a part of a DataStore,
in which case "stix_data" is a direct reference to
shared memory with DataSource. Not user supplied
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
allow_custom (bool): whether to allow custom objects/properties
when exporting STIX content to file.
Default: True.
Attributes:
_data (dict): the in-memory dict that holds STIX objects.
If apart of a MemoryStore, dict is shared between with
a MemorySource
If part of a MemoryStore, the dict is shared with a MemorySource
"""
def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False):
def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False):
super(MemorySink, self).__init__()
self._data = {}
self.allow_custom = allow_custom
if _store:
self._data = stix_data
elif stix_data:
_add(self, stix_data, allow_custom=allow_custom, version=version)
_add(self, stix_data, version=version)
def add(self, stix_data, allow_custom=False, version=None):
_add(self, stix_data, allow_custom=allow_custom, version=version)
def add(self, stix_data, version=None):
_add(self, stix_data, version=version)
add.__doc__ = _add.__doc__
def save_to_file(self, file_path, allow_custom=False):
def save_to_file(self, file_path):
file_path = os.path.abspath(file_path)
if not os.path.exists(os.path.dirname(file_path)):
os.makedirs(os.path.dirname(file_path))
with open(file_path, "w") as f:
f.write(str(Bundle(list(self._data.values()), allow_custom=allow_custom)))
f.write(str(Bundle(list(self._data.values()), allow_custom=self.allow_custom)))
save_to_file.__doc__ = MemoryStore.save_to_file.__doc__
@ -188,23 +173,24 @@ class MemorySource(DataSource):
_store (bool): if the MemorySource is a part of a DataStore,
in which case "stix_data" is a direct reference to shared
memory with DataSink. Not user supplied
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
allow_custom (bool): whether to allow custom objects/properties
when importing STIX content from file.
Default: True.
Attributes:
_data (dict): the in-memory dict that holds STIX objects.
If apart of a MemoryStore, dict is shared between with
a MemorySink
If part of a MemoryStore, the dict is shared with a MemorySink
"""
def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False):
def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False):
super(MemorySource, self).__init__()
self._data = {}
self.allow_custom = allow_custom
if _store:
self._data = stix_data
elif stix_data:
_add(self, stix_data, allow_custom=allow_custom, version=version)
_add(self, stix_data, version=version)
def get(self, stix_id, _composite_filters=None):
"""Retrieve STIX object from in-memory dict via STIX ID.
@ -260,6 +246,7 @@ class MemorySource(DataSource):
is returned in the same form as it as added
"""
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
def query(self, query=None, _composite_filters=None):
@ -301,8 +288,12 @@ class MemorySource(DataSource):
return all_data
def load_from_file(self, file_path, allow_custom=False, version=None):
file_path = os.path.abspath(file_path)
stix_data = json.load(open(file_path, "r"))
_add(self, stix_data, allow_custom=allow_custom, version=version)
def load_from_file(self, file_path, version=None):
stix_data = json.load(open(os.path.abspath(file_path), "r"))
if stix_data["type"] == "bundle":
for stix_obj in stix_data["objects"]:
_add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=stix_data["spec_version"]))
else:
_add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=version))
load_from_file.__doc__ = MemoryStore.load_from_file.__doc__

86
stix2/sources/taxii.py
View File

@ -1,6 +1,7 @@
"""
Python STIX 2.x TAXIICollectionStore
"""
from requests.exceptions import HTTPError
from stix2.base import _STIXBase
from stix2.core import Bundle, parse
@ -18,11 +19,23 @@ class TAXIICollectionStore(DataStore):
Args:
collection (taxii2.Collection): TAXII Collection instance
allow_custom (bool): whether to allow custom STIX content to be
pushed/retrieved. Defaults to True for TAXIICollectionSource
side(retrieving data) and False for TAXIICollectionSink
side(pushing data). However, when parameter is supplied, it will
be applied to both TAXIICollectionSource/Sink.
"""
def __init__(self, collection):
def __init__(self, collection, allow_custom=None):
if allow_custom is None:
allow_custom_source = True
allow_custom_sink = False
else:
allow_custom_sink = allow_custom_source = allow_custom
super(TAXIICollectionStore, self).__init__(
source=TAXIICollectionSource(collection),
sink=TAXIICollectionSink(collection)
source=TAXIICollectionSource(collection, allow_custom=allow_custom_source),
sink=TAXIICollectionSink(collection, allow_custom=allow_custom_sink)
)
@ -32,48 +45,49 @@ class TAXIICollectionSink(DataSink):
Args:
collection (taxii2.Collection): TAXII2 Collection instance
allow_custom (bool): Whether to allow custom STIX content to be
added to the TAXIICollectionSink. Default: False
"""
def __init__(self, collection):
def __init__(self, collection, allow_custom=False):
super(TAXIICollectionSink, self).__init__()
self.collection = collection
self.allow_custom = allow_custom
def add(self, stix_data, allow_custom=False, version=None):
def add(self, stix_data, version=None):
"""Add/push STIX content to TAXII Collection endpoint
Args:
stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content
in a STIX object (or Bundle), STIX onject dict (or Bundle dict), or a STIX 2.0
json encoded string, or list of any of the following
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
"""
if isinstance(stix_data, _STIXBase):
# adding python STIX object
bundle = dict(Bundle(stix_data, allow_custom=allow_custom))
bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
elif isinstance(stix_data, dict):
# adding python dict (of either Bundle or STIX obj)
if stix_data["type"] == "bundle":
bundle = stix_data
else:
bundle = dict(Bundle(stix_data, allow_custom=allow_custom))
bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
elif isinstance(stix_data, list):
# adding list of something - recurse on each
for obj in stix_data:
self.add(obj, allow_custom=allow_custom, version=version)
self.add(obj, version=version)
elif isinstance(stix_data, str):
# adding json encoded string of STIX content
stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version)
if stix_data["type"] == "bundle":
bundle = dict(stix_data)
else:
bundle = dict(Bundle(stix_data, allow_custom=allow_custom))
bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
else:
raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle")
@ -87,13 +101,16 @@ class TAXIICollectionSource(DataSource):
Args:
collection (taxii2.Collection): TAXII Collection instance
allow_custom (bool): Whether to allow custom STIX content to be
added to the FileSystemSink. Default: True
"""
def __init__(self, collection):
def __init__(self, collection, allow_custom=True):
super(TAXIICollectionSource, self).__init__()
self.collection = collection
self.allow_custom = allow_custom
def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
def get(self, stix_id, version=None, _composite_filters=None):
"""Retrieve STIX object from local/remote STIX Collection
endpoint.
@ -101,8 +118,6 @@ class TAXIICollectionSource(DataSource):
stix_id (str): The STIX ID of the STIX object to be retrieved.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -121,12 +136,16 @@ class TAXIICollectionSource(DataSource):
# dont extract TAXII filters from query (to send to TAXII endpoint)
# as directly retrieveing a STIX object by ID
stix_objs = self.collection.get_object(stix_id)["objects"]
try:
stix_objs = self.collection.get_object(stix_id)["objects"]
stix_obj = list(apply_common_filters(stix_objs, query))
stix_obj = list(apply_common_filters(stix_objs, query))
except HTTPError:
# if resource not found or access is denied from TAXII server, return None
stix_obj = []
if len(stix_obj):
stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version)
stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version)
if stix_obj.id != stix_id:
# check - was added to handle erroneous TAXII servers
stix_obj = None
@ -135,7 +154,7 @@ class TAXIICollectionSource(DataSource):
return stix_obj
def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
def all_versions(self, stix_id, version=None, _composite_filters=None):
"""Retrieve STIX object from local/remote TAXII Collection
endpoint, all versions of it
@ -143,8 +162,6 @@ class TAXIICollectionSource(DataSource):
stix_id (str): The STIX ID of the STIX objects to be retrieved.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -158,17 +175,17 @@ class TAXIICollectionSource(DataSource):
Filter("match[version]", "=", "all")
]
all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters)
all_data = self.query(query=query, _composite_filters=_composite_filters)
# parse STIX objects from TAXII returned json
all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data]
all_data = [parse(stix_obj, allow_custom=self.allow_custom, version=version) for stix_obj in all_data]
# check - was added to handle erroneous TAXII servers
all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id]
return all_data_clean
def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
def query(self, query=None, version=None, _composite_filters=None):
"""Search and retreive STIX objects based on the complete query
A "complete query" includes the filters from the query, the filters
@ -179,8 +196,6 @@ class TAXIICollectionSource(DataSource):
query (list): list of filters to search on
_composite_filters (set): set of filters passed from the
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -209,16 +224,21 @@ class TAXIICollectionSource(DataSource):
taxii_filters = self._parse_taxii_filters(query)
# query TAXII collection
all_data = self.collection.get_objects(filters=taxii_filters)["objects"]
try:
all_data = self.collection.get_objects(filters=taxii_filters)["objects"]
# deduplicate data (before filtering as reduces wasted filtering)
all_data = deduplicate(all_data)
# deduplicate data (before filtering as reduces wasted filtering)
all_data = deduplicate(all_data)
# apply local (CompositeDataSource, TAXIICollectionSource and query filters)
all_data = list(apply_common_filters(all_data, query))
# apply local (CompositeDataSource, TAXIICollectionSource and query filters)
all_data = list(apply_common_filters(all_data, query))
except HTTPError:
# if resources not found or access is denied from TAXII server, return empty list
all_data = []
# parse python STIX objects from the STIX object dicts
stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data]
stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data]
return stix_objs

10
stix2/test/test_filesystem.py
View File

@ -364,7 +364,7 @@ def test_filesystem_object_with_custom_property(fs_store):
fs_store.add(camp, True)
camp_r = fs_store.get(camp.id, allow_custom=True)
camp_r = fs_store.get(camp.id)
assert camp_r.id == camp.id
assert camp_r.x_empire == camp.x_empire
@ -376,9 +376,9 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store):
allow_custom=True)
bundle = Bundle(camp, allow_custom=True)
fs_store.add(bundle, allow_custom=True)
fs_store.add(bundle)
camp_r = fs_store.get(camp.id, allow_custom=True)
camp_r = fs_store.get(camp.id)
assert camp_r.id == camp.id
assert camp_r.x_empire == camp.x_empire
@ -391,9 +391,9 @@ def test_filesystem_custom_object(fs_store):
pass
newobj = NewObj(property1='something')
fs_store.add(newobj, allow_custom=True)
fs_store.add(newobj)
newobj_r = fs_store.get(newobj.id, allow_custom=True)
newobj_r = fs_store.get(newobj.id)
assert newobj_r.id == newobj.id
assert newobj_r.property1 == 'something'

55
stix2/test/test_memory.py
View File

@ -203,63 +203,12 @@ def test_memory_store_save_load_file(mem_store):
shutil.rmtree(os.path.dirname(filename))
def test_memory_store_add_stix_object_str(mem_store):
# add stix object string
camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a"
camp_name = "Aurelius"
camp_alias = "Purple Robes"
camp = """{
"name": "%s",
"type": "campaign",
"objective": "German and French Intelligence Services",
"aliases": ["%s"],
"id": "%s",
"created": "2017-05-31T21:31:53.197755Z"
}""" % (camp_name, camp_alias, camp_id)
mem_store.add(camp)
camp_r = mem_store.get(camp_id)
assert camp_r["id"] == camp_id
assert camp_r["name"] == camp_name
assert camp_alias in camp_r["aliases"]
def test_memory_store_add_stix_bundle_str(mem_store):
# add stix bundle string
camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a"
camp_name = "Atilla"
camp_alias = "Huns"
bund = """{
"type": "bundle",
"id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a",
"spec_version": "2.0",
"objects": [
{
"name": "%s",
"type": "campaign",
"objective": "Bulgarian, Albanian and Romanian Intelligence Services",
"aliases": ["%s"],
"id": "%s",
"created": "2017-05-31T21:31:53.197755Z"
}
]
}""" % (camp_name, camp_alias, camp_id)
mem_store.add(bund)
camp_r = mem_store.get(camp_id)
assert camp_r["id"] == camp_id
assert camp_r["name"] == camp_name
assert camp_alias in camp_r["aliases"]
def test_memory_store_add_invalid_object(mem_store):
ind = ('indicator', IND1) # tuple isn't valid
with pytest.raises(TypeError) as excinfo:
mem_store.add(ind)
assert 'stix_data must be' in str(excinfo.value)
assert 'a STIX object' in str(excinfo.value)
assert 'stix_data expected to be' in str(excinfo.value)
assert 'a python-stix2 object' in str(excinfo.value)
assert 'JSON formatted STIX' in str(excinfo.value)
assert 'JSON formatted STIX bundle' in str(excinfo.value)

42
stix2/test/test_versioning.py
View File

@ -88,11 +88,15 @@ def test_versioning_error_bad_modified_value():
assert excinfo.value.cls == stix2.Campaign
assert excinfo.value.prop_name == "modified"
assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime."
assert excinfo.value.reason == "The new modified datetime cannot be before than or equal to the current modified datetime." \
"It cannot be equal, as according to STIX 2 specification, objects that are different " \
"but have the same id and modified timestamp do not have defined consumer behavior."
msg = "Invalid value for {0} '{1}': {2}"
msg = msg.format(stix2.Campaign.__name__, "modified",
"The new modified datetime cannot be before the current modified datatime.")
"The new modified datetime cannot be before than or equal to the current modified datetime."
"It cannot be equal, as according to STIX 2 specification, objects that are different "
"but have the same id and modified timestamp do not have defined consumer behavior.")
assert str(excinfo.value) == msg
@ -153,7 +157,9 @@ def test_versioning_error_dict_bad_modified_value():
assert excinfo.value.cls == dict
assert excinfo.value.prop_name == "modified"
assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime."
assert excinfo.value.reason == "The new modified datetime cannot be before than or equal to the current modified datetime." \
"It cannot be equal, as according to STIX 2 specification, objects that are different " \
"but have the same id and modified timestamp do not have defined consumer behavior."
def test_versioning_error_dict_no_modified_value():
@ -206,3 +212,33 @@ def test_revoke_invalid_cls():
stix2.utils.revoke(campaign_v1)
assert 'cannot revoke object of this type' in str(excinfo.value)
def test_remove_custom_stix_property():
mal = stix2.Malware(name="ColePowers",
labels=["rootkit"],
x_custom="armada",
allow_custom=True)
mal_nc = stix2.utils.remove_custom_stix(mal)
assert "x_custom" not in mal_nc
assert stix2.utils.parse_into_datetime(mal["modified"], precision="millisecond") < stix2.utils.parse_into_datetime(mal_nc["modified"],
precision="millisecond")
def test_remove_custom_stix_object():
@stix2.CustomObject("x-animal", [
("species", stix2.properties.StringProperty(required=True)),
("animal_class", stix2.properties.StringProperty()),
])
class Animal(object):
def __init__(self, animal_class=None, **kwargs):
if animal_class and animal_class not in ["mammal", "bird"]:
raise ValueError("Not a recognized class of animal")
animal = Animal(species="lion", animal_class="mammal")
nc = stix2.utils.remove_custom_stix(animal)
assert nc is None

76
stix2/utils.py
View File

@ -1,5 +1,4 @@
"""Utility functions and classes for the stix2 library."""
from collections import Mapping
import copy
import datetime as dt
@ -16,6 +15,9 @@ from .exceptions import (InvalidValueError, RevokeError,
# timestamps in a single object, the timestamps will vary by a few microseconds.
NOW = object()
# STIX object properties that cannot be modified
STIX_UNMOD_PROPERTIES = ["created", "created_by_ref", "id", "type"]
class STIXdatetime(dt.datetime):
def __new__(cls, *args, **kwargs):
@ -215,7 +217,7 @@ def new_version(data, **kwargs):
properties_to_change = kwargs.keys()
# Make sure certain properties aren't trying to change
for prop in ["created", "created_by_ref", "id", "type"]:
for prop in STIX_UNMOD_PROPERTIES:
if prop in properties_to_change:
unchangable_properties.append(prop)
if unchangable_properties:
@ -227,8 +229,11 @@ def new_version(data, **kwargs):
elif 'modified' in data:
old_modified_property = parse_into_datetime(data.get('modified'), precision='millisecond')
new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond')
if new_modified_property < old_modified_property:
raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.")
if new_modified_property <= old_modified_property:
raise InvalidValueError(cls, 'modified',
"The new modified datetime cannot be before than or equal to the current modified datetime."
"It cannot be equal, as according to STIX 2 specification, objects that are different "
"but have the same id and modified timestamp do not have defined consumer behavior.")
new_obj_inner.update(kwargs)
# Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass
return cls(**{k: v for k, v in new_obj_inner.items() if v is not None})
@ -257,5 +262,68 @@ def get_class_hierarchy_names(obj):
return names
def remove_custom_stix(stix_obj):
"""remove any custom STIX objects or properties
Warning: This function is a best effort utility, in that
it will remove custom objects and properties based on the
type names; i.e. if "x-" prefixes object types, and "x_"
prefixes property types. According to the STIX2 spec,
those naming conventions are a SHOULDs not MUSTs, meaning
that valid custom STIX content may ignore those conventions
and in effect render this utility function invalid when used
on that STIX content.
Args:
stix_obj (dict OR python-stix obj): a single python-stix object
or dict of a STIX object
Returns:
A new version of the object with any custom content removed
"""
if stix_obj["type"].startswith("x-"):
# if entire object is custom, discard
return None
custom_props = []
for prop in stix_obj.items():
if prop[0].startswith("x_"):
# for every custom property, record it and set value to None
# (so we can pass it to new_version() and it will be dropped)
custom_props.append((prop[0], None))
if custom_props:
# obtain set of object properties that can be transferred
# to a new object version. This is 1)custom props with their
# values set to None, and 2)any properties left that are not
# unmodifiable STIX properties or the "modified" property
# set of properties that are not supplied to new_version()
# to be used for updating properties. This includes unmodifiable
# properties (properties that new_version() just re-uses from the
# existing STIX object) and the "modified" property. We dont supply the
# "modified" property so that new_version() creates a new datetime
# value for this property
non_supplied_props = STIX_UNMOD_PROPERTIES + ["modified"]
props = [(prop, stix_obj[prop]) for prop in stix_obj if prop not in non_supplied_props]
# add to set the custom properties we want to get rid of (with their value=None)
props.extend(custom_props)
new_obj = new_version(stix_obj, **(dict(props)))
while parse_into_datetime(new_obj["modified"]) == parse_into_datetime(stix_obj["modified"]):
# Prevents bug when fast computation allows multiple STIX object
# versions to be created in single unit of time
new_obj = new_version(stix_obj, **(dict(props)))
return new_obj
else:
return stix_obj
def get_type_from_id(stix_id):
return stix_id.split('--', 1)[0]