From 0d80d5fdfa3ece513bba17f097738570f5b5143b Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 19 Dec 2019 17:06:23 +0100 Subject: [PATCH 01/50] fix: [doc] Added APIVoid logo --- doc/logos/apivoid.png | Bin 0 -> 6955 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/logos/apivoid.png diff --git a/doc/logos/apivoid.png b/doc/logos/apivoid.png new file mode 100644 index 0000000000000000000000000000000000000000..e4f84a7c70228f40b8e69d8aafa565becd01ba5a GIT binary patch literal 6955 zcmZvBWmFVg)HO&AJvgM~0D^RP58VhzcRJD~9m>$1Qo{gBDBUT@&|ONW#3*XMMFd9e5(Bb&z|CduaPer+KW?F z1v!1c#r?%Q?c6ss11FC-Ie&KMHM6PtOT4_=RLBWAYpj6@0?ej^^FrCM7S5b3v3jV6 zHy?|wSr4x@!60^;18e%F&e{{4ie9}r>tBCo4U{zmFzTyWf5c^x%g-W3gOUEH*6qIRs_lrf8XN1sY?6-ZDUL@cL3)B=gRMFw zXZuU#2Fxygw_{1#s5UcKd$ed^X58#-ehcbS>cj}S9BuK*#Bh$YSU-h`1REdM-#6@* z{cE$CrZ>m5{ybyR>T2QdqFv;@6Yar64$IF}4`=$k+uNZ5keJNcV8D@=wCtOdab~dS z=vsNIpN~iQaEw78&&;|Hn`&g~PD#~)>D2^Qz1R+X_ZvM7=n3_U+EkX=Bzzbs92E#s zU}vIbB*2@CuUM@1%s(MC&Sr5u8s^>dZX+4Y>hrvj|K{gVb&H0=H6k1E zu?I@_o$S;=n0U8cE%A=D$-xm$w%w>qjF8Ck*wH=lH7HNW)R<}TCsu}QQ7DQlP@3fo zNHBgJ*z{;$5k#dZ{&5#&n+8LS$RA^m^%N*cU<{u$m+4DtiRv2s2K*>AlTt^tK6q)w z)e5_OfqnohL{q`4jTYkUxQXfeD|>YeGpeyVIR5JWem363?g^}v=t5Lffyo# zQ1hTq3xWE%V2%8l<+3u;k%B# zxsKYGmn&N{sgo9C+uJGS*>nU-HQgC6{T#@dA(cZ7dd^E)syh7uwU&VBBZVn((@WbQhS zZu#@wlbM({&sil<59Gfx`cLk0O;cU8>r&7Odq2K>jT2lnC8*B1V3a!l+x59EC$;zM zF91NaZblJ+Pp($sCAk7t-$(pt(u~zW$DRDZL)y@_BfI$#xYrQ@Q~N?N<~K9XhjSpIZ0d6AIJ8#n#(7eboF;t zRN#?iyn{loa%#ul)jkp*t+j+$>gZXzg=%;&I#sD}2tUJs{jtR#ubLvZz5_X)5cM2J4;6dOtfLA(+Fa;%_8&GuGF2|0DIC+R%P4M}!%9eZ zlj{y?yuUqI&*^5`jmn#?SVb?6+}XM=IXL_bN!dcuO;A-rCgZr{-0bggT>N=pvje>=jhBUr?~e*FTY!>!5!7IZDSFb^drH3 zY7`~%)%eAZa%lJ)BvNYDo}qsiNl~t9`7Ua3H}P*GeQh!c2Ggz5YlM4-o<=3&1X}G@_(5aR!WX zjfQ@3EO#J2U z;F7?YwrB?5a5_o9{X#1;>|G=ziHtP;*PPJYoCpP()B=6}H*rsSpBCeyg{b6VJ3-2f zz)35Wc*{qw>)XbT-QCFkNb(A8ntz9-*=|*RU16vf5OK9W&`C}g8?vttuwQ3me>xs4v628rd_FB^3VYtW245uAVF5`+O z4mO;N;`fz0)bcLbQn@j!pc#*Hk4=#Lb@mHt6V&J<#|&Od$uQ!lu2V*af!5hQ>_ov_ zp@9EHvWrKQNg*onyE2k)0IurHSFoc-3osYwb9{b1A)+xaHpTh=%V->;}nTZ`JrqyKjKM zdtVEq)OIV#hyw{T5C1ir!g7;oA+6}&SoA5l19Fi*1_)+P$;wWt^W)1GC&?j$;L$!( zDJmfUD044D&a@$)1>U^5^osMdx@(J4YIm?hEWOWmnJg{sijkyMju08-&lGrzfYt3ETORiXqQJrR z)!o;$zapqtp_mVopN54c&p|wD(_5Whn$wpC60eS$cu2j0(RZ{PGaA1UTez{cuqcxg z{ig4yOF+vS*+O7^POx~IwP-jdbrM#(AU1D*EM)VnvfT~Kx`HCsuH>`Gr5^IN1pg&I z>nzgP6*)xM!eb8sE4lgqS@$B=D8U2Zf}=L4`Dl^efD{6;#BdCL+!Bp!I;Q(tH*BBm zw?>SPC3gM0nWW^>MeiddW7Z^We_M{Up+sRqLJ4{-RW{Fkc5Cqo?&B91#cNY!!`%f{ zTS&OE+hYUGdkf@1iQI)4=07{zUi7wzxzp9@5OWdT0k-dq+n8PK@2{_0n3RkCd3R2pZ1OrB0A~B z_R=LBh_nMr%ck&K4cFqoz0%+(Sj>f8U?bqQEe-%xvG*3SOvHYV%Y@jFNv!@dDKL{y zhtQsvL-%AIHIXMK^7n7gE!`4jIKoox01xPDMqdW2=?V=KZ?{-iF8u>n;Pb1X{KJ?H z6#4Z2*I6p3AMZPM5Rq2%H!_<@_cA<&_|sI0XgOCwnFQRySjvcc(+y4$u+K|=FtIU1 z{7+LcSm5+6HCOQwTTms z${l@+090vw?lfxyRv{@mKCuy+9Stx2$P(FmEUO7PJFKJLsR5$G?iG=YTZD+SgVf+y zWDDY<3(GaWvt$NL$F#kt?gjLl6Hfe zedqu3l_wCHnMtHU$|q{AL=gw-aIvo0m42U%y-}8$ow@L_I<963tD1Z>$!-0v-Ou#h zD{{t;bq9NN2f#}LG;k1bhTU`B>HYzf&MG|5BFJiR>=lmEsYLUAoDw2c15N`TbYzWc zSS$4=U^;K?AE@=_zmy8n6#l?vSdGldF-eihz!+?rB^rwI4{%Z+_#LqyO{y$jQy{|$ z)1~1QtFqji!0i579FH&7L0CAfb@cYU;pyhdh<^xL{DkEG1U~a1xnGGW-v|nA#$!7W zI=>FEZsw4z9GwW4Ou0zsO`tmLyG6^#@OtY{R8G9YrAzTL3~uIN9B zJHW#$!PttfA_QsU{_PgMJ}wF~9{1GK+wV_aklt+>=~MR15486q~kR73CdI z=4@15NakkbZUa@cCBC*zAeQ9ASFP?(7^H1^uG5P=$KD7kxLurw#ROdh+n5g89*wQp z;Xu-V!?_`*o9(vI)%hAzYtlqwzB66Wv5> z92}SrSJtR_&y-fi$OL`6wictAl<4D}s?5b&3!;t8rWP%OM33S2E(qm-_xHBaX7N_V zt#JsrmhRvAtNSLPm{zxK|3PdsXM*b^;I>@SKf+oIq^nKk=5;m0zkkW9IPfn1;CA|2 z{S~7!chRHX7P@sAKps=qQjLKMc5mS|FBkqVUlkL$_$#HietIyOZ&{g*O3U0_>RL{) zVyb+<=yCH-VAbgCutSsATlB=>LUzW`66WpWBf|j$8VKnp>4bURdp>rES$6yzchKnB zee8%{c-132zqF*BXa1QA0%@spmbMaqFu(zIoZRl0z<39pu1!LlL6h*2SSF?L=a6!f zPLS#wy4v#L;Wnz7*|H?;(vV1~#_upSb&j3U)o6AmUfGY9bqtgB^3j-a^sY8Hh9ZQ> z*IW%3I26!9vo`ZmnC_8A}n1x><>G3r??UU|)dgEPXAiN364 zuV>bg?aaBo!FU9el958IZY+kToQlu5IAKd>e-EMFccKzS+&>aHc8!RC42#B*<= zi7WHl-K6JCdI(uZ2@e;7RXR_w=5Ux2H>IyQkpyBacAmKwdZA)g)jcX{Yfwqr zihPBS?wCNHf#&`qav9YBAtVqO5a{l!ow9HDygld8ALNF^|Jk7^ zz9|>~Ht}j=>W8s+JN36S^nk;Dt2sY_Wc%u@%Q&_W1vdpAF^-`=G`M*f#hn@B?#Li*7cz@ z>_Mu!vtu%aXcrOkkFJxgQ{swdFoxmYxc||js`Vqs2-r7-LG0@#=M_#>Te1Lwd$JUO z&nqqNNB^o;RaLK%Cw%#aPBm#+qC3OEi!jN3tybGX*2hx>zO1#m8=H+%P;S8=aJlh8n)&!8=t1aj3fV5k0NB#G@-ypOa1(TBg=Tre1~VjENL9I zk&m7KB;T2u-X)>9OW~5N3zf?q>6?F10#)&xR&RV<^z`CRLwqz>TJa5QP$MY}2M4qs zQEAkbmhp`xgEJ-ft@i>9uD z1B)5Amx8T5N+&Gz4aD-2`K@=CM=P9J5_gyS`*$X0-KxqeGGo(SX=&lgsWUHLfC6!i~79~a8Zr+Rxv9Wpti=9P=>h$J^ z9WO8W!{O#9F>Bppb29r%mj}DM;thWr|Gh(LAacY%i|&X1)P`TGFF_l*3v?B*>_7zV z4C=tAWe2kNKu=LrP?-EUm?*2`j$vQ?!@TSMUMOb{V_=qOubXZd*bY8n5k z6}2q#xtp)=Wk$4}dUvQI2|1kfMK&b}p%gFkse0DHtP83!&d%1=*>)LnI2PnEj%Dll z-okoesTqHSqkQ0FQGdT>nuiB8n~F}x|9w}ZmyKr*`c|7ou#^9`xz7QuE2G{_ee2E3 z(upm7gI{`d2cd>y{-2OV@vcvg*0Z=g6TgO(l-%50sqf)Y9rA;o&Ba_Sj|4I%MrNHx zCMkKDdS<%ad7-$>4FFc8&rsw1V@(ug87yBp+?3wi!+wBDzyY-ry11=M)w*wfc&~g9pDNWAsEc*qG16fk?xUWV6%q7G`r-9VUIC)? zekvLIrR`GqkXEGkG*JwTf;%@-xm9y=U@_GODtkQCsmyYqEqETm7$J z_hx1q64BxDXDf~EBqZ0g#D2>yFL%08w2vO%>W+l9w)!*Yy@WX^8d;4)W|QcJ-@_9fipeTxEa{IuO!8HDK0(t-*`K-znpUxX4<%tY16~_``sfy z#Q;|<{(SU47$<2%Op3jWaA_Mg{DHlYDNheKr6&D{;5{iWYpKK4{?*GJx1O3w6(*)7 ziGY9r9KVO%_YG}TwFhz(-JxnKD?L)$N9&Vi3clK*8U-PYV30{U=y@H1JjU|Wf}yU3 zJwUroO;kYQYKKscG-v8Tmw2JWvyw$rFcBhkdye(ohUGBBqKaREmEJj;U4{B>y?Vz zT7hC2tq;H19tF4g_AN`~@6>W3p9D->TaXw^-N-dYQ-!t3Agk@Rq(>S%r)_^ZPCq}r z;p|Qd1El4s)Su@db#@cyk<*i0<5%U}kFi<(X7+>_6$8B=hj+~{&Sqrg^})8rI5>;b zzEoZiMoaEf=j7yn$~A0iX-G|ZBYR*&VWQwURh=UutTs}v65K1&(AMN(zis~Bm5(?r zs1S+9a|C)LL>O;PK}i`jIm@+#KLyw=)9_e{lvZf z!ip{VxZa(c(`34E!6uUmH+gHORJ)@dZ&(6>)>Fo%uFgSVn}$Vcz;ASn+0PrBK}Zi;2zOlOLLgBJ4AO!rA>1iz7i?G{`@su?nH1v8 z@x+gMf|)&g;>$n5xIZwHBKgKYO*#v6=oI`I>L#iO+L%6ZMHQY@p8xNO|K|Yk)9$Ay eTeV~NgbS~@sjQ+HTc19`&{P$*6sjOrAO8oXq(346 literal 0 HcmV?d00001 From cf5ad29f270c5ade9d63b8f47d2a41366ff04168 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 7 Jan 2020 17:03:10 +0100 Subject: [PATCH 02/50] chg: Checking attributes category - We check the category before adding the attribute to the event - Checking if the category is correct and if not, doing a case insensitive check - If the category is not correct after the 2 first tests, we simply delete it from the attribute and pymisp will give the attribute a default category value based on the atttribute type, at the creation of the attribute --- misp_modules/modules/import_mod/csvimport.py | 67 ++++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 96e42b1..8bfbbe9 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -34,7 +34,7 @@ misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fi class CsvParser(): - def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes): + def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes, categories): self.misp_event = MISPEvent() self.header = header self.has_header = has_header @@ -42,11 +42,16 @@ class CsvParser(): self.data = data self.from_misp = from_misp self.MISPtypes = MISPtypes + self.categories = categories self.fields_number = len(self.header) - self.__score_mapping = {0: self.__create_standard_misp, + self.__score_mapping = {0: self.__create_standard_attribute, 1: self.__create_attribute_with_ids, 2: self.__create_attribute_with_tags, - 3: self.__create_attribute_with_ids_and_tags} + 3: self.__create_attribute_with_ids_and_tags, + 4: self.__create_attribute_check_category, + 5: self.__create_attribute_check_category_and_ids, + 6: self.__create_attribute_check_category_and_tags, + 7: self.__create_attribute_check_category_with_ids_and_tags} def parse_csv(self): if self.from_misp: @@ -165,35 +170,68 @@ class CsvParser(): # Utility functions # ################################################################################ + def __create_attribute_check_category(self, line, indexes): + attribute = self.__create_standard_attribute(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_and_ids(self, line, indexes): + attribute = self.__create_attribute_with_ids(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_and_tags(self, line, indexes): + attribute = self.__create_attribute_with_tags(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_with_ids_and_tags(self, line, indexes): + attribute = self.__create_attribute_with_ids_and_tags(line, indexes) + self.__check_category(attribute) + return attribute + def __create_attribute_with_ids(self, line, indexes): - attribute = self.__create_standard_misp(line, indexes) - return self.__deal_with_ids(attribute) + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_ids(attribute) + return attribute def __create_attribute_with_ids_and_tags(self, line, indexes): - attribute = self.__deal_with_ids(self.__create_standard_misp(line, indexes)) - return self.__deal_with_tags(attribute) + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_ids(attribute) + self.__deal_with_tags(attribute) + return attribute def __create_attribute_with_tags(self, line, indexes): - attribute = self.__create_standard_misp(line, indexes) - return self.__deal_with_tags(attribute) + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_tags(attribute) + return attribute - def __create_standard_misp(self, line, indexes): + def __create_standard_attribute(self, line, indexes): return {self.header[index]: line[index] for index in indexes if line[index]} + def __check_category(self, attribute): + category = attribute['category'] + if category in self.categories: + return + if category.capitalize() in self.categories: + attribute['category'] = category.capitalize() + return + del attribute['category'] + @staticmethod def __deal_with_ids(attribute): attribute['to_ids'] = True if attribute['to_ids'] == '1' else False - return attribute @staticmethod def __deal_with_tags(attribute): attribute['Tag'] = [{'name': tag.strip()} for tag in attribute['Tag'].split(',')] - return attribute def __get_score(self): score = 1 if 'to_ids' in self.header else 0 if 'attribute_tag' in self.header: score += 2 + if 'category' in self.header: + score += 4 return score def __finalize_results(self): @@ -241,7 +279,8 @@ def handler(q=False): header = misp_standard_csv_header descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json') with open(descFilename, 'r') as f: - MISPtypes = json.loads(f.read())['result'].get('types') + description = json.loads(f.read())['result'] + MISPtypes = description['types'] for h in header: if not any((h in MISPtypes, h in misp_extended_csv_header, h in ('', ' ', '_', 'object_id'))): misperrors['error'] = 'Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively skip it using a whitespace).'.format(h) @@ -256,7 +295,7 @@ def handler(q=False): wrong_types = tuple(wrong_type for wrong_type in ('type', 'value') if wrong_type in header) misperrors['error'] = 'Error with the following header: {}. It contains the following field(s): {}, which is(are) already provided by the usage of at least on MISP attribute type in the header.'.format(header, 'and'.join(wrong_types)) return misperrors - csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, MISPtypes) + csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, MISPtypes, description['categories']) # build the attributes result = csv_parser.parse_csv() if 'error' in result: From bfcba18e3c0e682512504aa6d82990a83a1adde6 Mon Sep 17 00:00:00 2001 From: Erick Cheng Date: Tue, 7 Jan 2020 18:58:40 +0100 Subject: [PATCH 03/50] Update ipasn.py --- misp_modules/modules/expansion/ipasn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index 8489aa0..cfdbaf5 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -28,7 +28,7 @@ def handler(q=False): if not values: misperrors['error'] = 'Unable to find the history of this IP' return misperrors - return {'results': [{'types': mispattributes['output'], 'values': values}]} + return {'results': [{'types': mispattributes['output'], 'values': [str(values)]}]} def introspection(): From 10b4e78704c273974f0cd9cb4a85678fbb53775c Mon Sep 17 00:00:00 2001 From: Alvaro Garcia Date: Thu, 9 Jan 2020 09:57:46 +0000 Subject: [PATCH 04/50] add vt_graph export module --- misp_modules/lib/vt_graph_parser/__init__.py | 12 + misp_modules/lib/vt_graph_parser/errors.py | 20 ++ .../lib/vt_graph_parser/helpers/__init__.py | 4 + .../lib/vt_graph_parser/helpers/parsers.py | 89 +++++ .../lib/vt_graph_parser/helpers/rules.py | 304 ++++++++++++++++++ .../lib/vt_graph_parser/helpers/wrappers.py | 59 ++++ .../lib/vt_graph_parser/importers/__init__.py | 12 + .../lib/vt_graph_parser/importers/base.py | 98 ++++++ .../importers/pymisp_response.py | 75 +++++ misp_modules/modules/export_mod/__init__.py | 2 +- misp_modules/modules/export_mod/vt_graph.py | 113 +++++++ 11 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 misp_modules/lib/vt_graph_parser/__init__.py create mode 100644 misp_modules/lib/vt_graph_parser/errors.py create mode 100644 misp_modules/lib/vt_graph_parser/helpers/__init__.py create mode 100644 misp_modules/lib/vt_graph_parser/helpers/parsers.py create mode 100644 misp_modules/lib/vt_graph_parser/helpers/rules.py create mode 100644 misp_modules/lib/vt_graph_parser/helpers/wrappers.py create mode 100644 misp_modules/lib/vt_graph_parser/importers/__init__.py create mode 100644 misp_modules/lib/vt_graph_parser/importers/base.py create mode 100644 misp_modules/lib/vt_graph_parser/importers/pymisp_response.py create mode 100644 misp_modules/modules/export_mod/vt_graph.py diff --git a/misp_modules/lib/vt_graph_parser/__init__.py b/misp_modules/lib/vt_graph_parser/__init__.py new file mode 100644 index 0000000..2a4d339 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/__init__.py @@ -0,0 +1,12 @@ +"""vt_graph_parser. + +This module provides methods to import graph from misp. +""" + + +from lib.vt_graph_parser.importers import from_pymisp_response + + +__all__ = [ + "from_pymisp_response" +] diff --git a/misp_modules/lib/vt_graph_parser/errors.py b/misp_modules/lib/vt_graph_parser/errors.py new file mode 100644 index 0000000..4063933 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/errors.py @@ -0,0 +1,20 @@ +"""vt_graph_parser.errors. + +This module provides custom errors for data importers. +""" + + +class GraphImportError(Exception): + pass + + +class InvalidFileFormatError(Exception): + pass + + +class MispEventNotFoundError(Exception): + pass + + +class ServerError(Exception): + pass diff --git a/misp_modules/lib/vt_graph_parser/helpers/__init__.py b/misp_modules/lib/vt_graph_parser/helpers/__init__.py new file mode 100644 index 0000000..336faee --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/__init__.py @@ -0,0 +1,4 @@ +"""vt_graph_parser.helpers. + +This modules provides functions and attributes to help MISP importers. +""" diff --git a/misp_modules/lib/vt_graph_parser/helpers/parsers.py b/misp_modules/lib/vt_graph_parser/helpers/parsers.py new file mode 100644 index 0000000..ef78313 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/parsers.py @@ -0,0 +1,89 @@ +"""vt_graph_parser.helpers.parsers. + +This module provides parsers for MISP inputs. +""" + + +from lib.vt_graph_parser.helpers.wrappers import MispAttribute + + +MISP_INPUT_ATTR = [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "url", + "filename|md5", + "filename", + "target-user", + "target-email" +] + +VIRUSTOTAL_GRAPH_LINK_PREFIX = "https://www.virustotal.com/graph/" + + +def _parse_data(attributes, objects): + """Parse MISP event attributes and objects data. + + Args: + attributes (dict): dictionary which contains the MISP event attributes data. + objects (dict): dictionary which contains the MISP event objects data. + + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + attributes_data = [] + vt_graph_link = "" + + # Get simple MISP event attributes. + attributes_data += ( + [attr for attr in attributes + if attr.get("type") in MISP_INPUT_ATTR]) + + # Get attributes from MISP objects too. + if objects: + for object_ in objects: + object_attrs = object_.get("Attribute", []) + attributes_data += ( + [attr for attr in object_attrs + if attr.get("type") in MISP_INPUT_ATTR]) + + # Check if there is any VirusTotal Graph computed in MISP event. + vt_graph_links = ( + attr for attr in attributes if attr.get("type") == "link" + and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) + + # MISP could have more than one VirusTotal Graph, so we will take + # the last one. + current_id = 0 # MISP attribute id is the number of the attribute. + vt_graph_link = "" + for link in vt_graph_links: + if int(link.get("id")) > current_id: + current_id = int(link.get("id")) + vt_graph_link = link.get("value") + + attributes = [ + MispAttribute(data["type"], data["category"], data["value"]) + for data in attributes_data] + return (attributes, + vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) + + +def parse_pymisp_response(payload): + """Get event attributes and VirusTotal Graph id from pymisp response. + + Args: + payload (dict): dictionary which contains pymisp response. + + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + event_attrs = payload.get("Attribute", []) + objects = payload.get("Object") + return _parse_data(event_attrs, objects) + diff --git a/misp_modules/lib/vt_graph_parser/helpers/rules.py b/misp_modules/lib/vt_graph_parser/helpers/rules.py new file mode 100644 index 0000000..14230d0 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/rules.py @@ -0,0 +1,304 @@ +"""vt_graph_parser.helpers.rules. + +This module provides rules that helps MISP importers to connect MISP attributes +between them using VirusTotal relationship. Check all available relationship +here: + +- File: https://developers.virustotal.com/v3/reference/#files-relationships +- URL: https://developers.virustotal.com/v3/reference/#urls-relationships +- Domain: https://developers.virustotal.com/v3/reference/#domains-relationships +- IP: https://developers.virustotal.com/v3/reference/#ip-relationships +""" + + +import abc + + +class MispEventRule(object): + """Rules for MISP event nodes connection object wrapper.""" + + def __init__(self, last_rule=None, node=None): + """Create a MispEventRule instance. + + MispEventRule is a collection of rules that can infer the relationships + between nodes from MISP events. + + Args: + last_rule (MispEventRule): previous rule. + node (Node): actual node. + """ + self.last_rule = last_rule + self.node = node + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def get_last_different_rule(self): + """Search the last rule whose event was different from actual. + + Returns: + MispEventRule: the last different rule. + """ + if not isinstance(self, self.last_rule.__class__): + return self.last_rule + else: + return self.last_rule.get_last_different_rule() + + def resolve_relation(self, graph, node, misp_category): + """Try to infer a relationship between two nodes. + + This method is based on a non-deterministic finite automaton for + this reason the future rule only depends on the actual rule and the input + node. + + For example if the actual rule is a MISPEventDomainRule and the given node + is an ip_address node, the connection type between them will be + `resolutions` and the this rule will transit to MISPEventIPRule. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + if node.node_type in self.relation_event: + return self.relation_event[node.node_type](graph, node, misp_category) + else: + return self.manual_link(graph, node) + + def manual_link(self, graph, node): + """Creates a manual link between self.node and the given node. + + We accept MISP types that VirusTotal does not know how to link, so we create + a end to end relationship instead of create an unknown relationship node. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + + Returns: + MispEventRule: the transited rule. + """ + graph.add_link(self.node.node_id, node.node_id, "manual") + return self + + @abc.abstractmethod + def __file_transition(self, graph, node, misp_category): + """Make a new transition due to file attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __ip_transition(self, graph, node, misp_category): + """Make a new transition due to ip attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __url_transition(self, graph, node, misp_category): + """Make a new transition due to url attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __domain_transition(self, graph, node, misp_category): + """Make a new transition due to domain attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + +class MispEventURLRule(MispEventRule): + """Rule for URL event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventURLRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "downloaded_files") + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) + + +class MispEventIPRule(MispEventRule): + """Rule for IP event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventIPRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventDomainRule(self, node) + + +class MispEventDomainRule(MispEventRule): + """Rule for domain event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventDomainRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + graph.add_link(self.node.node_id, node.node_id, "siblings") + return MispEventDomainRule(self, node) + + +class MispEventFileRule(MispEventRule): + """Rule for File event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventFileRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) + + +class MispEventInitialRule(MispEventRule): + """Initial rule.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventInitialRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + return MispEventDomainRule(self, node) diff --git a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py new file mode 100644 index 0000000..8735317 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py @@ -0,0 +1,59 @@ +"""vt_graph_parser.helpers.wrappers. + +This module provides a Python object wrapper for MISP objects. +""" + + +class MispAttribute(object): + """Python object wrapper for MISP attribute. + + Attributes: + type (str): VirusTotal node type. + category (str): MISP attribute category. + value (str): node id. + label (str): node name. + misp_type (str): MISP node type. + """ + + MISP_TYPES_REFERENCE = { + "hostname": "domain", + "domain": "domain", + "ip-src": "ip_address", + "ip-dst": "ip_address", + "url": "url", + "filename|X": "file", + "filename": "file", + "md5": "file", + "sha1": "file", + "sha256": "file", + "target-user": "victim", + "target-email": "email" + } + + def __init__(self, misp_type, category, value, label=""): + """Constructor for a MispAttribute. + + Args: + misp_type (str): MISP type attribute. + category (str): MISP category attribute. + value (str): attribute value. + label (str): attribute label. + """ + if misp_type.startswith("filename|"): + label, value = value.split("|") + misp_type = "filename|X" + if misp_type == "filename": + label = value + + self.type = self.MISP_TYPES_REFERENCE.get(misp_type) + self.category = category + self.value = value + self.label = label + self.misp_type = misp_type + + def __eq__(self, other): + return (isinstance(other, self.__class__) and self.value == other.value and + self.type == other.type) + + def __repr__(self): + return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) diff --git a/misp_modules/lib/vt_graph_parser/importers/__init__.py b/misp_modules/lib/vt_graph_parser/importers/__init__.py new file mode 100644 index 0000000..129d870 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/__init__.py @@ -0,0 +1,12 @@ +"""vt_graph_parser.importers. + +This module provides methods to import graphs from MISP. +""" + + +from lib.vt_graph_parser.importers.pymisp_response import from_pymisp_response + + +__all__ = [ + "from_pymisp_response" +] diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py new file mode 100644 index 0000000..cdea8b6 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -0,0 +1,98 @@ +"""vt_graph_parser.importers.base. + +This module provides a common method to import graph from misp attributes. +""" + + +import vt_graph_api +from lib.vt_graph_parser.helpers.rules import MispEventRule + + +def import_misp_graph( + misp_attributes, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth): + """Import VirusTotal Graph from MISP. + + Args: + misp_attributes ([MispAttribute]): list with the MISP attributes which + will be added to the returned graph. + graph_id: if supplied, the graph will be loaded instead of compute it again. + vt_api_key (str): VT API Key. + fetch_information (bool): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str): graph title. Defaults to "". + private (bool): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str]): usernames that can edit the graph. + Defaults to None. + user_viewers ([str]): usernames that can view the graph. + Defaults to None. + group_editors ([str]): groups that can edit the graph. + Defaults to None. + group_viewers ([str]): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. + + Returns: + vt_graph_api.graph.VTGraph: the imported graph. + """ + + rule = MispEventInitialRule() + + # Check if the event has been already computed in VirusTotal Graph. Otherwise + # a new graph will be created. + if not graph_id: + graph = vt_graph_api.VTGraph( + api_key=vt_api_key, name=name, private=private, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers) + else: + graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key) + + attributes_to_add = [attr for attr in misp_attributes + if not graph.has_node(attr.value)] + + total_expandable_attrs = max(sum( + 1 for attr in attributes_to_add + if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), + 1) + + max_quotas_per_search = max(int(max_api_quotas / total_expandable_attrs), 1) + + previous_node_id = "" + for attr in attributes_to_add: + # Add the current attr as node to the graph. + added_node = graph.add_node( + attr.value, attr.type, fetch_information, fetch_vt_enterprise, + attr.label) + # If use_vt_to_connect_the_grap is True the nodes will be connected using + # VT API. + if use_vt_to_connect_the_graph: + if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and + previous_node_id): + graph.add_link(previous_node_id, attr.value, "manual") + else: + graph.connect_with_graph( + attr.value, max_quotas_per_search, max_search_depth, + fetch_info_collected_nodes=fetch_information) + else: + rule = rule.resolve_relation(graph, added_node, attr.category) + + return graph diff --git a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py new file mode 100644 index 0000000..c01b6a1 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py @@ -0,0 +1,75 @@ +"""vt_graph_parser.importers.pymisp_response. + +This modules provides a graph importer method for MISP event by using the +response payload giving by MISP API directly. +""" + + +import json +from lib.vt_graph_parser import errors +from lib.vt_graph_parser.helpers.parsers import parse_pymisp_response +from lib.vt_graph_parser.importers.base import import_misp_graph + + +def from_pymisp_response( + payload, vt_api_key, fetch_information=True, + private=False, fetch_vt_enterprise=False, user_editors=None, + user_viewers=None, group_editors=None, group_viewers=None, + use_vt_to_connect_the_graph=False, max_api_quotas=1000, + max_search_depth=3, expand_node_one_level=False): + """Import VirusTotal Graph from MISP JSON file. + + Args: + payload (dict): dictionary which contains the request payload. + vt_api_key (str): VT API Key. + fetch_information (bool, optional): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str, optional): graph title. Defaults to "". + private (bool, optional): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str], optional): usernames that can edit the graph. + Defaults to None. + user_viewers ([str], optional): usernames that can view the graph. + Defaults to None. + group_editors ([str], optional): groups that can edit the graph. + Defaults to None. + group_viewers ([str], optional): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int, optional): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + expand_one_level (bool, optional): expand entire graph one level. + Defaults to False. + + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. + + Raises: + LoaderError: if JSON file is invalid. + + Returns: + [vt_graph_api.graph.VTGraph: the imported graph]. + """ + graphs = [] + for event_payload in payload['data']: + misp_attrs, graph_id = parse_pymisp_response(event_payload) + name = "Graph created from MISP event" + graph = import_misp_graph( + misp_attrs, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth) + if expand_node_one_level: + graph.expand_n_level(1) + graphs.append(graph) + return graphs diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 77dec0d..1b0e1d0 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ __all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', - 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] + 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph'] diff --git a/misp_modules/modules/export_mod/vt_graph.py b/misp_modules/modules/export_mod/vt_graph.py new file mode 100644 index 0000000..9d20a00 --- /dev/null +++ b/misp_modules/modules/export_mod/vt_graph.py @@ -0,0 +1,113 @@ +'''Export MISP event to VirusTotal Graph.''' + + +import base64 +import json +from lib.vt_graph_parser import from_pymisp_response + + +misperrors = { + 'error': 'Error' +} +moduleinfo = { + 'version': '0.1', + 'author': 'VirusTotal', + 'description': 'Send event to VirusTotal Graph', + 'module-type': ['export'] +} +mispattributes = { + 'input': [ + 'hostname', + 'domain', + 'ip-src', + 'ip-dst', + 'md5', + 'sha1', + 'sha256', + 'url', + 'filename|md5', + 'filename' + ] +} +moduleconfig = [ + 'vt_api_key', + 'fetch_information', + 'private', + 'fetch_vt_enterprise', + 'expand_one_level', + 'user_editors', + 'user_viewers', + 'group_editors', + 'group_viewers' +] + + +def handler(q=False): + """Expansion handler. + + Args: + q (bool, optional): module data. Defaults to False. + + Returns: + [str]: VirusTotal graph links + """ + if not q: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('vt_api_key'): + misperrors['error'] = 'A VirusTotal api key is required for this module.' + return misperrors + + config = request['config'] + + api_key = config.get('vt_api_key') + fetch_information = config.get('fetch_information') or False + private = config.get('private') or False + fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False + expand_one_level = config.get('expand_one_level') or False + + user_editors = config.get('user_editors') + if user_editors: + user_editors = user_editors.split(',') + user_viewers = config.get('user_viewers') + if user_viewers: + user_viewers = user_viewers.split(',') + group_editors = config.get('group_editors') + if group_editors: + group_editors = group_editors.split(',') + group_viewers = config.get('group_viewers') + if group_viewers: + group_viewers = group_viewers.split(',') + + + graphs = from_pymisp_response( + request, api_key, fetch_information=fetch_information, + private=private, fetch_vt_enterprise=fetch_vt_enterprise, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers, + expand_node_one_level=expand_one_level) + links = [] + + for graph in graphs: + graph.save_graph() + links.append(graph.get_ui_link()) + + # This file will contains one VirusTotal graph link for each exported event + file_data = str(base64.b64encode(bytes('\n'.join(links), 'utf-8')), 'utf-8') + return {'response': [], 'data': file_data} + + +def introspection(): + modulesetup = { + 'responseType': 'application/txt', + 'outputFileExtension': 'txt', + 'userConfig': {}, + 'inputSource': [] + } + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 3207ceca046c3bb3faf296ee35ca1fe2785df2fc Mon Sep 17 00:00:00 2001 From: Alvaro Garcia Date: Thu, 9 Jan 2020 12:39:43 +0000 Subject: [PATCH 05/50] Add vt-graph-api to the requirements --- Pipfile | 1 + REQUIREMENTS | 1 + 2 files changed, 2 insertions(+) diff --git a/Pipfile b/Pipfile index 1a99c42..9e651de 100644 --- a/Pipfile +++ b/Pipfile @@ -59,6 +59,7 @@ jbxapi = "*" geoip2 = "*" apiosintDS = "*" assemblyline_client = "*" +vt-graph-api = "*" [requires] python_version = "3" diff --git a/REQUIREMENTS b/REQUIREMENTS index ee6c7c1..40b4caf 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -106,3 +106,4 @@ xlsxwriter==1.2.6 yara-python==3.8.1 yarl==1.4.2 zipp==0.6.0 +vt-graph-api From 7722e2cb93014c6e9c2c59f8939029084995ff79 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 9 Jan 2020 15:28:33 +0100 Subject: [PATCH 06/50] fix: Fixed typo on function import --- misp_modules/lib/vt_graph_parser/importers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py index cdea8b6..3cd0192 100644 --- a/misp_modules/lib/vt_graph_parser/importers/base.py +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -5,7 +5,7 @@ This module provides a common method to import graph from misp attributes. import vt_graph_api -from lib.vt_graph_parser.helpers.rules import MispEventRule +from lib.vt_graph_parser.helpers.rules import MispEventInitialRule def import_misp_graph( From 70b3079aa3a9eacb7348090da7a0a18d199f605f Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 9 Jan 2020 16:01:18 +0100 Subject: [PATCH 07/50] fix: Fixed pep8 in the new module and related libraries --- misp_modules/lib/vt_graph_parser/errors.py | 8 +- .../lib/vt_graph_parser/helpers/parsers.py | 97 ++-- .../lib/vt_graph_parser/helpers/rules.py | 442 +++++++++--------- .../lib/vt_graph_parser/helpers/wrappers.py | 93 ++-- .../lib/vt_graph_parser/importers/base.py | 160 +++---- .../importers/pymisp_response.py | 116 +++-- misp_modules/modules/export_mod/vt_graph.py | 106 ++--- 7 files changed, 509 insertions(+), 513 deletions(-) diff --git a/misp_modules/lib/vt_graph_parser/errors.py b/misp_modules/lib/vt_graph_parser/errors.py index 4063933..a7e18e9 100644 --- a/misp_modules/lib/vt_graph_parser/errors.py +++ b/misp_modules/lib/vt_graph_parser/errors.py @@ -5,16 +5,16 @@ This module provides custom errors for data importers. class GraphImportError(Exception): - pass + pass class InvalidFileFormatError(Exception): - pass + pass class MispEventNotFoundError(Exception): - pass + pass class ServerError(Exception): - pass + pass diff --git a/misp_modules/lib/vt_graph_parser/helpers/parsers.py b/misp_modules/lib/vt_graph_parser/helpers/parsers.py index ef78313..c621595 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/parsers.py +++ b/misp_modules/lib/vt_graph_parser/helpers/parsers.py @@ -26,64 +26,63 @@ VIRUSTOTAL_GRAPH_LINK_PREFIX = "https://www.virustotal.com/graph/" def _parse_data(attributes, objects): - """Parse MISP event attributes and objects data. + """Parse MISP event attributes and objects data. - Args: - attributes (dict): dictionary which contains the MISP event attributes data. - objects (dict): dictionary which contains the MISP event objects data. + Args: + attributes (dict): dictionary which contains the MISP event attributes data. + objects (dict): dictionary which contains the MISP event objects data. - Returns: - ([MispAttribute], str): MISP attributes and VTGraph link if exists. - Link defaults to "". - """ - attributes_data = [] - vt_graph_link = "" + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + attributes_data = [] + vt_graph_link = "" - # Get simple MISP event attributes. - attributes_data += ( - [attr for attr in attributes - if attr.get("type") in MISP_INPUT_ATTR]) + # Get simple MISP event attributes. + attributes_data += ( + [attr for attr in attributes + if attr.get("type") in MISP_INPUT_ATTR]) - # Get attributes from MISP objects too. - if objects: - for object_ in objects: - object_attrs = object_.get("Attribute", []) - attributes_data += ( - [attr for attr in object_attrs - if attr.get("type") in MISP_INPUT_ATTR]) + # Get attributes from MISP objects too. + if objects: + for object_ in objects: + object_attrs = object_.get("Attribute", []) + attributes_data += ( + [attr for attr in object_attrs + if attr.get("type") in MISP_INPUT_ATTR]) - # Check if there is any VirusTotal Graph computed in MISP event. - vt_graph_links = ( - attr for attr in attributes if attr.get("type") == "link" - and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) + # Check if there is any VirusTotal Graph computed in MISP event. + vt_graph_links = ( + attr for attr in attributes if attr.get("type") == "link" + and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) - # MISP could have more than one VirusTotal Graph, so we will take - # the last one. - current_id = 0 # MISP attribute id is the number of the attribute. - vt_graph_link = "" - for link in vt_graph_links: - if int(link.get("id")) > current_id: - current_id = int(link.get("id")) - vt_graph_link = link.get("value") + # MISP could have more than one VirusTotal Graph, so we will take + # the last one. + current_id = 0 # MISP attribute id is the number of the attribute. + vt_graph_link = "" + for link in vt_graph_links: + if int(link.get("id")) > current_id: + current_id = int(link.get("id")) + vt_graph_link = link.get("value") - attributes = [ - MispAttribute(data["type"], data["category"], data["value"]) - for data in attributes_data] - return (attributes, - vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) + attributes = [ + MispAttribute(data["type"], data["category"], data["value"]) + for data in attributes_data] + return (attributes, + vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) def parse_pymisp_response(payload): - """Get event attributes and VirusTotal Graph id from pymisp response. + """Get event attributes and VirusTotal Graph id from pymisp response. - Args: - payload (dict): dictionary which contains pymisp response. - - Returns: - ([MispAttribute], str): MISP attributes and VTGraph link if exists. - Link defaults to "". - """ - event_attrs = payload.get("Attribute", []) - objects = payload.get("Object") - return _parse_data(event_attrs, objects) + Args: + payload (dict): dictionary which contains pymisp response. + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + event_attrs = payload.get("Attribute", []) + objects = payload.get("Object") + return _parse_data(event_attrs, objects) diff --git a/misp_modules/lib/vt_graph_parser/helpers/rules.py b/misp_modules/lib/vt_graph_parser/helpers/rules.py index 14230d0..e3ed7f8 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/rules.py +++ b/misp_modules/lib/vt_graph_parser/helpers/rules.py @@ -15,290 +15,290 @@ import abc class MispEventRule(object): - """Rules for MISP event nodes connection object wrapper.""" + """Rules for MISP event nodes connection object wrapper.""" - def __init__(self, last_rule=None, node=None): - """Create a MispEventRule instance. + def __init__(self, last_rule=None, node=None): + """Create a MispEventRule instance. - MispEventRule is a collection of rules that can infer the relationships - between nodes from MISP events. + MispEventRule is a collection of rules that can infer the relationships + between nodes from MISP events. - Args: - last_rule (MispEventRule): previous rule. - node (Node): actual node. - """ - self.last_rule = last_rule - self.node = node - self.relation_event = { - "ip_address": self.__ip_transition, - "url": self.__url_transition, - "domain": self.__domain_transition, - "file": self.__file_transition - } + Args: + last_rule (MispEventRule): previous rule. + node (Node): actual node. + """ + self.last_rule = last_rule + self.node = node + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } - def get_last_different_rule(self): - """Search the last rule whose event was different from actual. + def get_last_different_rule(self): + """Search the last rule whose event was different from actual. - Returns: - MispEventRule: the last different rule. - """ - if not isinstance(self, self.last_rule.__class__): - return self.last_rule - else: - return self.last_rule.get_last_different_rule() + Returns: + MispEventRule: the last different rule. + """ + if not isinstance(self, self.last_rule.__class__): + return self.last_rule + else: + return self.last_rule.get_last_different_rule() - def resolve_relation(self, graph, node, misp_category): - """Try to infer a relationship between two nodes. + def resolve_relation(self, graph, node, misp_category): + """Try to infer a relationship between two nodes. - This method is based on a non-deterministic finite automaton for - this reason the future rule only depends on the actual rule and the input - node. + This method is based on a non-deterministic finite automaton for + this reason the future rule only depends on the actual rule and the input + node. - For example if the actual rule is a MISPEventDomainRule and the given node - is an ip_address node, the connection type between them will be - `resolutions` and the this rule will transit to MISPEventIPRule. + For example if the actual rule is a MISPEventDomainRule and the given node + is an ip_address node, the connection type between them will be + `resolutions` and the this rule will transit to MISPEventIPRule. - Args: - graph (VTGraph): graph to be computed. - node (Node): the node to be linked. - misp_category: (str): MISP category of the given node. + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. - Returns: - MispEventRule: the transited rule. - """ - if node.node_type in self.relation_event: - return self.relation_event[node.node_type](graph, node, misp_category) - else: - return self.manual_link(graph, node) + Returns: + MispEventRule: the transited rule. + """ + if node.node_type in self.relation_event: + return self.relation_event[node.node_type](graph, node, misp_category) + else: + return self.manual_link(graph, node) - def manual_link(self, graph, node): - """Creates a manual link between self.node and the given node. + def manual_link(self, graph, node): + """Creates a manual link between self.node and the given node. - We accept MISP types that VirusTotal does not know how to link, so we create - a end to end relationship instead of create an unknown relationship node. + We accept MISP types that VirusTotal does not know how to link, so we create + a end to end relationship instead of create an unknown relationship node. - Args: - graph (VTGraph): graph to be computed. - node (Node): the node to be linked. + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. - Returns: - MispEventRule: the transited rule. - """ - graph.add_link(self.node.node_id, node.node_id, "manual") - return self + Returns: + MispEventRule: the transited rule. + """ + graph.add_link(self.node.node_id, node.node_id, "manual") + return self - @abc.abstractmethod - def __file_transition(self, graph, node, misp_category): - """Make a new transition due to file attribute event. + @abc.abstractmethod + def __file_transition(self, graph, node, misp_category): + """Make a new transition due to file attribute event. - Args: - graph (VTGraph): graph to be computed. - node (Node): the node to be linked. - misp_category: (str): MISP category of the given node. + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. - Returns: - MispEventRule: the transited rule. - """ - pass + Returns: + MispEventRule: the transited rule. + """ + pass - @abc.abstractmethod - def __ip_transition(self, graph, node, misp_category): - """Make a new transition due to ip attribute event. + @abc.abstractmethod + def __ip_transition(self, graph, node, misp_category): + """Make a new transition due to ip attribute event. - Args: - graph (VTGraph): graph to be computed. - node (Node): the node to be linked. - misp_category: (str): MISP category of the given node. + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. - Returns: - MispEventRule: the transited rule. - """ - pass + Returns: + MispEventRule: the transited rule. + """ + pass - @abc.abstractmethod - def __url_transition(self, graph, node, misp_category): - """Make a new transition due to url attribute event. + @abc.abstractmethod + def __url_transition(self, graph, node, misp_category): + """Make a new transition due to url attribute event. - Args: - graph (VTGraph): graph to be computed. - node (Node): the node to be linked. - misp_category: (str): MISP category of the given node. + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. - Returns: - MispEventRule: the transited rule. - """ - pass + Returns: + MispEventRule: the transited rule. + """ + pass - @abc.abstractmethod - def __domain_transition(self, graph, node, misp_category): - """Make a new transition due to domain attribute event. + @abc.abstractmethod + def __domain_transition(self, graph, node, misp_category): + """Make a new transition due to domain attribute event. - Args: - graph (VTGraph): graph to be computed. - node (Node): the node to be linked. - misp_category: (str): MISP category of the given node. + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. - Returns: - MispEventRule: the transited rule. - """ - pass + Returns: + MispEventRule: the transited rule. + """ + pass class MispEventURLRule(MispEventRule): - """Rule for URL event.""" + """Rule for URL event.""" - def __init__(self, last_rule=None, node=None): - super(MispEventURLRule, self).__init__(last_rule, node) - self.relation_event = { - "ip_address": self.__ip_transition, - "url": self.__url_transition, - "domain": self.__domain_transition, - "file": self.__file_transition - } + def __init__(self, last_rule=None, node=None): + super(MispEventURLRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } - def __file_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "downloaded_files") - return MispEventFileRule(self, node) + def __file_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "downloaded_files") + return MispEventFileRule(self, node) - def __ip_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "contacted_ips") - return MispEventIPRule(self, node) + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) - def __url_transition(self, graph, node, misp_category): - suitable_rule = self.get_last_different_rule() - if not isinstance(suitable_rule, MispEventInitialRule): - return suitable_rule.resolve_relation(graph, node, misp_category) - else: - return MispEventURLRule(self, node) + def __url_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventURLRule(self, node) - def __domain_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "contacted_domains") - return MispEventDomainRule(self, node) + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) class MispEventIPRule(MispEventRule): - """Rule for IP event.""" + """Rule for IP event.""" - def __init__(self, last_rule=None, node=None): - super(MispEventIPRule, self).__init__(last_rule, node) - self.relation_event = { - "ip_address": self.__ip_transition, - "url": self.__url_transition, - "domain": self.__domain_transition, - "file": self.__file_transition - } + def __init__(self, last_rule=None, node=None): + super(MispEventIPRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } - def __file_transition(self, graph, node, misp_category): - connection_type = "communicating_files" - if misp_category == "Artifacts dropped": - connection_type = "downloaded_files" - graph.add_link(self.node.node_id, node.node_id, connection_type) - return MispEventFileRule(self, node) + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) - def __ip_transition(self, graph, node, misp_category): - suitable_rule = self.get_last_different_rule() - if not isinstance(suitable_rule, MispEventInitialRule): - return suitable_rule.resolve_relation(graph, node, misp_category) - else: - return MispEventIPRule(self, node) + def __ip_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventIPRule(self, node) - def __url_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "urls") - return MispEventURLRule(self, node) + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) - def __domain_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "resolutions") - return MispEventDomainRule(self, node) + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventDomainRule(self, node) class MispEventDomainRule(MispEventRule): - """Rule for domain event.""" + """Rule for domain event.""" - def __init__(self, last_rule=None, node=None): - super(MispEventDomainRule, self).__init__(last_rule, node) - self.relation_event = { - "ip_address": self.__ip_transition, - "url": self.__url_transition, - "domain": self.__domain_transition, - "file": self.__file_transition - } + def __init__(self, last_rule=None, node=None): + super(MispEventDomainRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } - def __file_transition(self, graph, node, misp_category): - connection_type = "communicating_files" - if misp_category == "Artifacts dropped": - connection_type = "downloaded_files" - graph.add_link(self.node.node_id, node.node_id, connection_type) - return MispEventFileRule(self, node) + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) - def __ip_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "resolutions") - return MispEventIPRule(self, node) + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventIPRule(self, node) - def __url_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "urls") - return MispEventURLRule(self, node) + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) - def __domain_transition(self, graph, node, misp_category): - suitable_rule = self.get_last_different_rule() - if not isinstance(suitable_rule, MispEventInitialRule): - return suitable_rule.resolve_relation(graph, node, misp_category) - else: - graph.add_link(self.node.node_id, node.node_id, "siblings") - return MispEventDomainRule(self, node) + def __domain_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + graph.add_link(self.node.node_id, node.node_id, "siblings") + return MispEventDomainRule(self, node) class MispEventFileRule(MispEventRule): - """Rule for File event.""" + """Rule for File event.""" - def __init__(self, last_rule=None, node=None): - super(MispEventFileRule, self).__init__(last_rule, node) - self.relation_event = { - "ip_address": self.__ip_transition, - "url": self.__url_transition, - "domain": self.__domain_transition, - "file": self.__file_transition - } + def __init__(self, last_rule=None, node=None): + super(MispEventFileRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } - def __file_transition(self, graph, node, misp_category): - suitable_rule = self.get_last_different_rule() - if not isinstance(suitable_rule, MispEventInitialRule): - return suitable_rule.resolve_relation(graph, node, misp_category) - else: - return MispEventFileRule(self, node) + def __file_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventFileRule(self, node) - def __ip_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "contacted_ips") - return MispEventIPRule(self, node) + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) - def __url_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "contacted_urls") - return MispEventURLRule(self, node) + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_urls") + return MispEventURLRule(self, node) - def __domain_transition(self, graph, node, misp_category): - graph.add_link(self.node.node_id, node.node_id, "contacted_domains") - return MispEventDomainRule(self, node) + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) class MispEventInitialRule(MispEventRule): - """Initial rule.""" + """Initial rule.""" - def __init__(self, last_rule=None, node=None): - super(MispEventInitialRule, self).__init__(last_rule, node) - self.relation_event = { - "ip_address": self.__ip_transition, - "url": self.__url_transition, - "domain": self.__domain_transition, - "file": self.__file_transition - } + def __init__(self, last_rule=None, node=None): + super(MispEventInitialRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } - def __file_transition(self, graph, node, misp_category): - return MispEventFileRule(self, node) + def __file_transition(self, graph, node, misp_category): + return MispEventFileRule(self, node) - def __ip_transition(self, graph, node, misp_category): - return MispEventIPRule(self, node) + def __ip_transition(self, graph, node, misp_category): + return MispEventIPRule(self, node) - def __url_transition(self, graph, node, misp_category): - return MispEventURLRule(self, node) + def __url_transition(self, graph, node, misp_category): + return MispEventURLRule(self, node) - def __domain_transition(self, graph, node, misp_category): - return MispEventDomainRule(self, node) + def __domain_transition(self, graph, node, misp_category): + return MispEventDomainRule(self, node) diff --git a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py index 8735317..d376d43 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py +++ b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py @@ -5,55 +5,54 @@ This module provides a Python object wrapper for MISP objects. class MispAttribute(object): - """Python object wrapper for MISP attribute. + """Python object wrapper for MISP attribute. - Attributes: - type (str): VirusTotal node type. - category (str): MISP attribute category. - value (str): node id. - label (str): node name. - misp_type (str): MISP node type. - """ - - MISP_TYPES_REFERENCE = { - "hostname": "domain", - "domain": "domain", - "ip-src": "ip_address", - "ip-dst": "ip_address", - "url": "url", - "filename|X": "file", - "filename": "file", - "md5": "file", - "sha1": "file", - "sha256": "file", - "target-user": "victim", - "target-email": "email" - } - - def __init__(self, misp_type, category, value, label=""): - """Constructor for a MispAttribute. - - Args: - misp_type (str): MISP type attribute. - category (str): MISP category attribute. - value (str): attribute value. - label (str): attribute label. + Attributes: + type (str): VirusTotal node type. + category (str): MISP attribute category. + value (str): node id. + label (str): node name. + misp_type (str): MISP node type. """ - if misp_type.startswith("filename|"): - label, value = value.split("|") - misp_type = "filename|X" - if misp_type == "filename": - label = value - self.type = self.MISP_TYPES_REFERENCE.get(misp_type) - self.category = category - self.value = value - self.label = label - self.misp_type = misp_type + MISP_TYPES_REFERENCE = { + "hostname": "domain", + "domain": "domain", + "ip-src": "ip_address", + "ip-dst": "ip_address", + "url": "url", + "filename|X": "file", + "filename": "file", + "md5": "file", + "sha1": "file", + "sha256": "file", + "target-user": "victim", + "target-email": "email" + } - def __eq__(self, other): - return (isinstance(other, self.__class__) and self.value == other.value and - self.type == other.type) + def __init__(self, misp_type, category, value, label=""): + """Constructor for a MispAttribute. - def __repr__(self): - return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) + Args: + misp_type (str): MISP type attribute. + category (str): MISP category attribute. + value (str): attribute value. + label (str): attribute label. + """ + if misp_type.startswith("filename|"): + label, value = value.split("|") + misp_type = "filename|X" + if misp_type == "filename": + label = value + + self.type = self.MISP_TYPES_REFERENCE.get(misp_type) + self.category = category + self.value = value + self.label = label + self.misp_type = misp_type + + def __eq__(self, other): + return (isinstance(other, self.__class__) and self.value == other.value and self.type == other.type) + + def __repr__(self): + return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py index 3cd0192..4d9b855 100644 --- a/misp_modules/lib/vt_graph_parser/importers/base.py +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -9,90 +9,90 @@ from lib.vt_graph_parser.helpers.rules import MispEventInitialRule def import_misp_graph( - misp_attributes, graph_id, vt_api_key, fetch_information, name, - private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, - group_viewers, use_vt_to_connect_the_graph, max_api_quotas, - max_search_depth): - """Import VirusTotal Graph from MISP. + misp_attributes, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth): + """Import VirusTotal Graph from MISP. - Args: - misp_attributes ([MispAttribute]): list with the MISP attributes which - will be added to the returned graph. - graph_id: if supplied, the graph will be loaded instead of compute it again. - vt_api_key (str): VT API Key. - fetch_information (bool): whether the script will fetch - information for added nodes in VT. Defaults to True. - name (str): graph title. Defaults to "". - private (bool): True for private graphs. You need to have - Private Graph premium features enabled in your subscription. Defaults - to False. - fetch_vt_enterprise (bool, optional): if True, the graph will search any - available information using VirusTotal Intelligence for the node if there - is no normal information for it. Defaults to False. - user_editors ([str]): usernames that can edit the graph. - Defaults to None. - user_viewers ([str]): usernames that can view the graph. - Defaults to None. - group_editors ([str]): groups that can edit the graph. - Defaults to None. - group_viewers ([str]): groups that can view the graph. - Defaults to None. - use_vt_to_connect_the_graph (bool): if True, graph nodes will - be linked using VirusTotal API. Otherwise, the links will be generated - using production rules based on MISP attributes order. Defaults to - False. - max_api_quotas (int): maximum number of api quotas that could - be consumed to resolve graph using VirusTotal API. Defaults to 20000. - max_search_depth (int, optional): max search depth to explore - relationship between nodes when use_vt_to_connect_the_graph is True. - Defaults to 3. + Args: + misp_attributes ([MispAttribute]): list with the MISP attributes which + will be added to the returned graph. + graph_id: if supplied, the graph will be loaded instead of compute it again. + vt_api_key (str): VT API Key. + fetch_information (bool): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str): graph title. Defaults to "". + private (bool): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str]): usernames that can edit the graph. + Defaults to None. + user_viewers ([str]): usernames that can view the graph. + Defaults to None. + group_editors ([str]): groups that can edit the graph. + Defaults to None. + group_viewers ([str]): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. - If use_vt_to_connect_the_graph is True, it will take some time to compute - graph. + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. - Returns: - vt_graph_api.graph.VTGraph: the imported graph. - """ + Returns: + vt_graph_api.graph.VTGraph: the imported graph. + """ - rule = MispEventInitialRule() + rule = MispEventInitialRule() - # Check if the event has been already computed in VirusTotal Graph. Otherwise - # a new graph will be created. - if not graph_id: - graph = vt_graph_api.VTGraph( - api_key=vt_api_key, name=name, private=private, - user_editors=user_editors, user_viewers=user_viewers, - group_editors=group_editors, group_viewers=group_viewers) - else: - graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key) - - attributes_to_add = [attr for attr in misp_attributes - if not graph.has_node(attr.value)] - - total_expandable_attrs = max(sum( - 1 for attr in attributes_to_add - if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), - 1) - - max_quotas_per_search = max(int(max_api_quotas / total_expandable_attrs), 1) - - previous_node_id = "" - for attr in attributes_to_add: - # Add the current attr as node to the graph. - added_node = graph.add_node( - attr.value, attr.type, fetch_information, fetch_vt_enterprise, - attr.label) - # If use_vt_to_connect_the_grap is True the nodes will be connected using - # VT API. - if use_vt_to_connect_the_graph: - if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and - previous_node_id): - graph.add_link(previous_node_id, attr.value, "manual") - else: - graph.connect_with_graph( - attr.value, max_quotas_per_search, max_search_depth, - fetch_info_collected_nodes=fetch_information) + # Check if the event has been already computed in VirusTotal Graph. Otherwise + # a new graph will be created. + if not graph_id: + graph = vt_graph_api.VTGraph( + api_key=vt_api_key, name=name, private=private, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers) else: - rule = rule.resolve_relation(graph, added_node, attr.category) + graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key) - return graph + attributes_to_add = [attr for attr in misp_attributes + if not graph.has_node(attr.value)] + + total_expandable_attrs = max(sum( + 1 for attr in attributes_to_add + if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), + 1) + + max_quotas_per_search = max( + int(max_api_quotas / total_expandable_attrs), 1) + + previous_node_id = "" + for attr in attributes_to_add: + # Add the current attr as node to the graph. + added_node = graph.add_node( + attr.value, attr.type, fetch_information, fetch_vt_enterprise, + attr.label) + # If use_vt_to_connect_the_grap is True the nodes will be connected using + # VT API. + if use_vt_to_connect_the_graph: + if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and previous_node_id): + graph.add_link(previous_node_id, attr.value, "manual") + else: + graph.connect_with_graph( + attr.value, max_quotas_per_search, max_search_depth, + fetch_info_collected_nodes=fetch_information) + else: + rule = rule.resolve_relation(graph, added_node, attr.category) + + return graph diff --git a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py index c01b6a1..86a3b25 100644 --- a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +++ b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py @@ -5,71 +5,69 @@ response payload giving by MISP API directly. """ -import json -from lib.vt_graph_parser import errors from lib.vt_graph_parser.helpers.parsers import parse_pymisp_response from lib.vt_graph_parser.importers.base import import_misp_graph def from_pymisp_response( - payload, vt_api_key, fetch_information=True, - private=False, fetch_vt_enterprise=False, user_editors=None, - user_viewers=None, group_editors=None, group_viewers=None, - use_vt_to_connect_the_graph=False, max_api_quotas=1000, - max_search_depth=3, expand_node_one_level=False): - """Import VirusTotal Graph from MISP JSON file. + payload, vt_api_key, fetch_information=True, + private=False, fetch_vt_enterprise=False, user_editors=None, + user_viewers=None, group_editors=None, group_viewers=None, + use_vt_to_connect_the_graph=False, max_api_quotas=1000, + max_search_depth=3, expand_node_one_level=False): + """Import VirusTotal Graph from MISP JSON file. - Args: - payload (dict): dictionary which contains the request payload. - vt_api_key (str): VT API Key. - fetch_information (bool, optional): whether the script will fetch - information for added nodes in VT. Defaults to True. - name (str, optional): graph title. Defaults to "". - private (bool, optional): True for private graphs. You need to have - Private Graph premium features enabled in your subscription. Defaults - to False. - fetch_vt_enterprise (bool, optional): if True, the graph will search any - available information using VirusTotal Intelligence for the node if there - is no normal information for it. Defaults to False. - user_editors ([str], optional): usernames that can edit the graph. - Defaults to None. - user_viewers ([str], optional): usernames that can view the graph. - Defaults to None. - group_editors ([str], optional): groups that can edit the graph. - Defaults to None. - group_viewers ([str], optional): groups that can view the graph. - Defaults to None. - use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will - be linked using VirusTotal API. Otherwise, the links will be generated - using production rules based on MISP attributes order. Defaults to - False. - max_api_quotas (int, optional): maximum number of api quotas that could - be consumed to resolve graph using VirusTotal API. Defaults to 20000. - max_search_depth (int, optional): max search depth to explore - relationship between nodes when use_vt_to_connect_the_graph is True. - Defaults to 3. - expand_one_level (bool, optional): expand entire graph one level. - Defaults to False. + Args: + payload (dict): dictionary which contains the request payload. + vt_api_key (str): VT API Key. + fetch_information (bool, optional): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str, optional): graph title. Defaults to "". + private (bool, optional): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str], optional): usernames that can edit the graph. + Defaults to None. + user_viewers ([str], optional): usernames that can view the graph. + Defaults to None. + group_editors ([str], optional): groups that can edit the graph. + Defaults to None. + group_viewers ([str], optional): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int, optional): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + expand_one_level (bool, optional): expand entire graph one level. + Defaults to False. - If use_vt_to_connect_the_graph is True, it will take some time to compute - graph. + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. - Raises: - LoaderError: if JSON file is invalid. + Raises: + LoaderError: if JSON file is invalid. - Returns: - [vt_graph_api.graph.VTGraph: the imported graph]. - """ - graphs = [] - for event_payload in payload['data']: - misp_attrs, graph_id = parse_pymisp_response(event_payload) - name = "Graph created from MISP event" - graph = import_misp_graph( - misp_attrs, graph_id, vt_api_key, fetch_information, name, - private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, - group_viewers, use_vt_to_connect_the_graph, max_api_quotas, - max_search_depth) - if expand_node_one_level: - graph.expand_n_level(1) - graphs.append(graph) - return graphs + Returns: + [vt_graph_api.graph.VTGraph: the imported graph]. + """ + graphs = [] + for event_payload in payload['data']: + misp_attrs, graph_id = parse_pymisp_response(event_payload) + name = "Graph created from MISP event" + graph = import_misp_graph( + misp_attrs, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth) + if expand_node_one_level: + graph.expand_n_level(1) + graphs.append(graph) + return graphs diff --git a/misp_modules/modules/export_mod/vt_graph.py b/misp_modules/modules/export_mod/vt_graph.py index 9d20a00..d8b3359 100644 --- a/misp_modules/modules/export_mod/vt_graph.py +++ b/misp_modules/modules/export_mod/vt_graph.py @@ -43,71 +43,71 @@ moduleconfig = [ def handler(q=False): - """Expansion handler. + """Expansion handler. - Args: - q (bool, optional): module data. Defaults to False. + Args: + q (bool, optional): module data. Defaults to False. - Returns: - [str]: VirusTotal graph links - """ - if not q: - return False - request = json.loads(q) + Returns: + [str]: VirusTotal graph links + """ + if not q: + return False + request = json.loads(q) - if not request.get('config') or not request['config'].get('vt_api_key'): - misperrors['error'] = 'A VirusTotal api key is required for this module.' - return misperrors + if not request.get('config') or not request['config'].get('vt_api_key'): + misperrors['error'] = 'A VirusTotal api key is required for this module.' + return misperrors - config = request['config'] + config = request['config'] - api_key = config.get('vt_api_key') - fetch_information = config.get('fetch_information') or False - private = config.get('private') or False - fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False - expand_one_level = config.get('expand_one_level') or False + api_key = config.get('vt_api_key') + fetch_information = config.get('fetch_information') or False + private = config.get('private') or False + fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False + expand_one_level = config.get('expand_one_level') or False - user_editors = config.get('user_editors') - if user_editors: - user_editors = user_editors.split(',') - user_viewers = config.get('user_viewers') - if user_viewers: - user_viewers = user_viewers.split(',') - group_editors = config.get('group_editors') - if group_editors: - group_editors = group_editors.split(',') - group_viewers = config.get('group_viewers') - if group_viewers: - group_viewers = group_viewers.split(',') - + user_editors = config.get('user_editors') + if user_editors: + user_editors = user_editors.split(',') + user_viewers = config.get('user_viewers') + if user_viewers: + user_viewers = user_viewers.split(',') + group_editors = config.get('group_editors') + if group_editors: + group_editors = group_editors.split(',') + group_viewers = config.get('group_viewers') + if group_viewers: + group_viewers = group_viewers.split(',') - graphs = from_pymisp_response( - request, api_key, fetch_information=fetch_information, - private=private, fetch_vt_enterprise=fetch_vt_enterprise, - user_editors=user_editors, user_viewers=user_viewers, - group_editors=group_editors, group_viewers=group_viewers, - expand_node_one_level=expand_one_level) - links = [] + graphs = from_pymisp_response( + request, api_key, fetch_information=fetch_information, + private=private, fetch_vt_enterprise=fetch_vt_enterprise, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers, + expand_node_one_level=expand_one_level) + links = [] - for graph in graphs: - graph.save_graph() - links.append(graph.get_ui_link()) + for graph in graphs: + graph.save_graph() + links.append(graph.get_ui_link()) - # This file will contains one VirusTotal graph link for each exported event - file_data = str(base64.b64encode(bytes('\n'.join(links), 'utf-8')), 'utf-8') - return {'response': [], 'data': file_data} + # This file will contains one VirusTotal graph link for each exported event + file_data = str(base64.b64encode( + bytes('\n'.join(links), 'utf-8')), 'utf-8') + return {'response': [], 'data': file_data} def introspection(): - modulesetup = { - 'responseType': 'application/txt', - 'outputFileExtension': 'txt', - 'userConfig': {}, - 'inputSource': [] - } - return modulesetup + modulesetup = { + 'responseType': 'application/txt', + 'outputFileExtension': 'txt', + 'userConfig': {}, + 'inputSource': [] + } + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo From f197abdcf6e719671bb62bc0643eb2f9f65c4108 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 9 Jan 2020 16:04:29 +0100 Subject: [PATCH 08/50] chg: Bumped pipfile.lock with up-to-date libraries and new vt_graph_api library requirement --- Pipfile.lock | 342 ++++++++++++++++++++++++++------------------------- 1 file changed, 175 insertions(+), 167 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index dab4860..b977ce7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "30e84f4986146c248e706f52f425649660225889bfcdf5075c99854442ae5f42" + "sha256": "b62db6df8a7b42f4c6915d6fbb1d4c38ccbb7209e559708433d28cdddebd3df9" }, "pipfile-spec": 6, "requires": { @@ -96,12 +96,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", - "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931", - "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" + "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a", + "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887", + "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae" ], "index": "pypi", - "version": "==4.8.1" + "version": "==4.8.2" }, "blockchain": { "hashes": [ @@ -264,20 +264,28 @@ ], "version": "==0.18.2" }, + "futures": { + "hashes": [ + "sha256:3a44f286998ae64f0cc083682fcfec16c406134a81a589a5de445d7bb7c2751b", + "sha256:51ecb45f0add83c806c68e4b06106f90db260585b25ef2abfcda0bd95c0132fd", + "sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f" + ], + "version": "==3.1.1" + }, "geoip2": { "hashes": [ - "sha256:a37ddac2d200ffb97c736da8b8ba9d5d8dc47da6ec0f162a461b681ecac53a14", - "sha256:f7ffe9d258e71a42cf622ce6350d976de1d0312b9f2fbce3975c7d838b57ecf0" + "sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0", + "sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4" ], "index": "pypi", - "version": "==2.9.0" + "version": "==3.0.0" }, "httplib2": { "hashes": [ - "sha256:34537dcdd5e0f2386d29e0e2c6d4a1703a3b982d34c198a5102e6e5d6194b107", - "sha256:409fa5509298f739b34d5a652df762cb0042507dc93f6633e306b11289d6249d" + "sha256:1d1f4ad7a6e55d325830ab274190f98894e069850a871fac19921caf4363259d", + "sha256:a5f914f18f99cb9541660454a159e3b3c63241fc3ab60005bb88d97cc7a4fb58" ], - "version": "==0.14.0" + "version": "==0.15.0" }, "idna": { "hashes": [ @@ -384,9 +392,9 @@ }, "maxminddb": { "hashes": [ - "sha256:449a1713d37320d777d0db286286ab22890f0a176492ecf3ad8d9319108f2f79" + "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336" ], - "version": "==1.5.1" + "version": "==1.5.2" }, "misp-modules": { "editable": true, @@ -401,25 +409,25 @@ }, "multidict": { "hashes": [ - "sha256:09c19f642e055550c9319d5123221b7e07fc79bda58122aa93910e52f2ab2f29", - "sha256:0c1a5d5f7aa7189f7b83c4411c2af8f1d38d69c4360d5de3eea129c65d8d7ce2", - "sha256:12f22980e7ed0972a969520fb1e55682c9fca89a68b21b49ec43132e680be812", - "sha256:258660e9d6b52de1a75097944e12718d3aa59adc611b703361e3577d69167aaf", - "sha256:3374a23e707848f27b3438500db0c69eca82929337656fce556bd70031fbda74", - "sha256:503b7fce0054c73aa631cc910a470052df33d599f3401f3b77e54d31182525d5", - "sha256:6ce55f2c45ffc90239aab625bb1b4864eef33f73ea88487ef968291fbf09fb3f", - "sha256:725496dde5730f4ad0a627e1a58e2620c1bde0ad1c8080aae15d583eb23344ce", - "sha256:a3721078beff247d0cd4fb19d915c2c25f90907cf8d6cd49d0413a24915577c6", - "sha256:ba566518550f81daca649eded8b5c7dd09210a854637c82351410aa15c49324a", - "sha256:c42362750a51a15dc905cb891658f822ee5021bfbea898c03aa1ed833e2248a5", - "sha256:cf14aaf2ab067ca10bca0b14d5cbd751dd249e65d371734bc0e47ddd8fafc175", - "sha256:cf24e15986762f0e75a622eb19cfe39a042e952b8afba3e7408835b9af2be4fb", - "sha256:d7b6da08538302c5245cd3103f333655ba7f274915f1f5121c4f4b5fbdb3febe", - "sha256:e27e13b9ff0a914a6b8fb7e4947d4ac6be8e4f61ede17edffabd088817df9e26", - "sha256:e53b205f8afd76fc6c942ef39e8ee7c519c775d336291d32874082a87802c67c", - "sha256:ec804fc5f68695d91c24d716020278fcffd50890492690a7e1fef2e741f7172c" + "sha256:0f04bf4c15d8417401a10a650c349ccc0285943681bfd87d3690587d7714a9b4", + "sha256:15a61c0df2d32487e06f6084eabb48fd9e8b848315e397781a70caf9670c9d78", + "sha256:3c5e2dcbe6b04cbb4303e47a896757a77b676c5e5db5528be7ff92f97ba7ab95", + "sha256:5d2b32b890d9e933d3ced417924261802a857abdee9507b68c75014482145c03", + "sha256:5e5fb8bfebf87f2e210306bf9dd8de2f1af6782b8b78e814060ae9254ab1f297", + "sha256:63ba2be08d82ea2aa8b0f7942a74af4908664d26cb4ff60c58eadb1e33e7da00", + "sha256:73740fcdb38f0adcec85e97db7557615b50ec4e5a3e73e35878720bcee963382", + "sha256:78bed18e7f1eb21f3d10ff3acde900b4d630098648fe1d65bb4abfb3e22c4900", + "sha256:a02fade7b5476c4f88efe9593ff2f3286698d8c6d715ba4f426954f73f382026", + "sha256:aacbde3a8875352a640efa2d1b96e5244a29b0f8df79cbf1ec6470e86fd84697", + "sha256:be813fb9e5ce41a5a99a29cdb857144a1bd6670883586f995b940a4878dc5238", + "sha256:bfcad6da0b8839f01a819602aaa5c5a5b4c85ecbfae9b261a31df3d9262fb31e", + "sha256:c2bfc0db3166e68515bc4a2b9164f4f75ae9c793e9635f8651f2c9ffc65c8dad", + "sha256:c66d11870ae066499a3541963e6ce18512ca827c2aaeaa2f4e37501cee39ac5d", + "sha256:cc7f2202b753f880c2e4123f9aacfdb94560ba893e692d24af271dac41f8b8d9", + "sha256:d1f45e5bb126662ba66ee579831ce8837b1fd978115c9657e32eb3c75b92973d", + "sha256:ed5f3378c102257df9e2dc9ce6468dabf68bee9ec34969cfdc472631aba00316" ], - "version": "==4.7.1" + "version": "==4.7.3" }, "np": { "hashes": [ @@ -430,29 +438,29 @@ }, "numpy": { "hashes": [ - "sha256:0a7a1dd123aecc9f0076934288ceed7fd9a81ba3919f11a855a7887cbe82a02f", - "sha256:0c0763787133dfeec19904c22c7e358b231c87ba3206b211652f8cbe1241deb6", - "sha256:3d52298d0be333583739f1aec9026f3b09fdfe3ddf7c7028cb16d9d2af1cca7e", - "sha256:43bb4b70585f1c2d153e45323a886839f98af8bfa810f7014b20be714c37c447", - "sha256:475963c5b9e116c38ad7347e154e5651d05a2286d86455671f5b1eebba5feb76", - "sha256:64874913367f18eb3013b16123c9fed113962e75d809fca5b78ebfbb73ed93ba", - "sha256:683828e50c339fc9e68720396f2de14253992c495fdddef77a1e17de55f1decc", - "sha256:6ca4000c4a6f95a78c33c7dadbb9495c10880be9c89316aa536eac359ab820ae", - "sha256:75fd817b7061f6378e4659dd792c84c0b60533e867f83e0d1e52d5d8e53df88c", - "sha256:7d81d784bdbed30137aca242ab307f3e65c8d93f4c7b7d8f322110b2e90177f9", - "sha256:8d0af8d3664f142414fd5b15cabfd3b6cc3ef242a3c7a7493257025be5a6955f", - "sha256:9679831005fb16c6df3dd35d17aa31dc0d4d7573d84f0b44cc481490a65c7725", - "sha256:a8f67ebfae9f575d85fa859b54d3bdecaeece74e3274b0b5c5f804d7ca789fe1", - "sha256:acbf5c52db4adb366c064d0b7c7899e3e778d89db585feadd23b06b587d64761", - "sha256:ada4805ed51f5bcaa3a06d3dd94939351869c095e30a2b54264f5a5004b52170", - "sha256:c7354e8f0eca5c110b7e978034cd86ed98a7a5ffcf69ca97535445a595e07b8e", - "sha256:e2e9d8c87120ba2c591f60e32736b82b67f72c37ba88a4c23c81b5b8fa49c018", - "sha256:e467c57121fe1b78a8f68dd9255fbb3bb3f4f7547c6b9e109f31d14569f490c3", - "sha256:ede47b98de79565fcd7f2decb475e2dcc85ee4097743e551fe26cfc7eb3ff143", - "sha256:f58913e9227400f1395c7b800503ebfdb0772f1c33ff8cb4d6451c06cabdf316", - "sha256:fe39f5fd4103ec4ca3cb8600b19216cd1ff316b4990f4c0b6057ad982c0a34d5" + "sha256:1786a08236f2c92ae0e70423c45e1e62788ed33028f94ca99c4df03f5be6b3c6", + "sha256:17aa7a81fe7599a10f2b7d95856dc5cf84a4eefa45bc96123cbbc3ebc568994e", + "sha256:20b26aaa5b3da029942cdcce719b363dbe58696ad182aff0e5dcb1687ec946dc", + "sha256:2d75908ab3ced4223ccba595b48e538afa5ecc37405923d1fea6906d7c3a50bc", + "sha256:39d2c685af15d3ce682c99ce5925cc66efc824652e10990d2462dfe9b8918c6a", + "sha256:56bc8ded6fcd9adea90f65377438f9fea8c05fcf7c5ba766bef258d0da1554aa", + "sha256:590355aeade1a2eaba17617c19edccb7db8d78760175256e3cf94590a1a964f3", + "sha256:70a840a26f4e61defa7bdf811d7498a284ced303dfbc35acb7be12a39b2aa121", + "sha256:77c3bfe65d8560487052ad55c6998a04b654c2fbc36d546aef2b2e511e760971", + "sha256:9537eecf179f566fd1c160a2e912ca0b8e02d773af0a7a1120ad4f7507cd0d26", + "sha256:9acdf933c1fd263c513a2df3dceecea6f3ff4419d80bf238510976bf9bcb26cd", + "sha256:ae0975f42ab1f28364dcda3dde3cf6c1ddab3e1d4b2909da0cb0191fa9ca0480", + "sha256:b3af02ecc999c8003e538e60c89a2b37646b39b688d4e44d7373e11c2debabec", + "sha256:b6ff59cee96b454516e47e7721098e6ceebef435e3e21ac2d6c3b8b02628eb77", + "sha256:b765ed3930b92812aa698a455847141869ef755a87e099fddd4ccf9d81fffb57", + "sha256:c98c5ffd7d41611407a1103ae11c8b634ad6a43606eca3e2a5a269e5d6e8eb07", + "sha256:cf7eb6b1025d3e169989416b1adcd676624c2dbed9e3bcb7137f51bfc8cc2572", + "sha256:d92350c22b150c1cae7ebb0ee8b5670cc84848f6359cf6b5d8f86617098a9b73", + "sha256:e422c3152921cece8b6a2fb6b0b4d73b6579bd20ae075e7d15143e711f3ca2ca", + "sha256:e840f552a509e3380b0f0ec977e8124d0dc34dc0e68289ca28f4d7c1d0d79474", + "sha256:f3d0a94ad151870978fb93538e95411c83899c9dc63e6fb65542f769568ecfa5" ], - "version": "==1.17.4" + "version": "==1.18.1" }, "oauth2": { "hashes": [ @@ -543,46 +551,38 @@ }, "pdftotext": { "hashes": [ - "sha256:c8bdc47b08baa17b8e03ba1f960fc6335b183d2644eaf7300e088516758a6090" + "sha256:b56f6ff1a564803ab8d849b3bb350b27087c15f5fe4e542a6370645543b0adf9" ], "index": "pypi", - "version": "==2.1.2" + "version": "==2.1.3" }, "pillow": { "hashes": [ - "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", - "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", - "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", - "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", - "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", - "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", - "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", - "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", - "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", - "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", - "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", - "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", - "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", - "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", - "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", - "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", - "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", - "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", - "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", - "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", - "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", - "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", - "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", - "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", - "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", - "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", - "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", - "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", - "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", - "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" + "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be", + "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946", + "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837", + "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f", + "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00", + "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d", + "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533", + "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a", + "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358", + "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda", + "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435", + "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2", + "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313", + "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff", + "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317", + "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2", + "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614", + "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0", + "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386", + "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9", + "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636", + "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865" ], "index": "pypi", - "version": "==6.2.1" + "version": "==7.0.0" }, "progressbar2": { "hashes": [ @@ -736,7 +736,7 @@ "fileobjects,openioc,virustotal,pdfexport" ], "git": "https://github.com/MISP/PyMISP.git", - "ref": "a26a8e450b14d48bb0c8ef46b32bff2f1eadc514" + "ref": "3ee7d8c67601bee658f1c0f488635796e5d7eb04" }, "pyonyphe": { "editable": true, @@ -752,10 +752,10 @@ }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "pypdns": { "hashes": [ @@ -774,16 +774,16 @@ }, "pyrsistent": { "hashes": [ - "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b" + "sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280" ], - "version": "==0.15.6" + "version": "==0.15.7" }, "pytesseract": { "hashes": [ - "sha256:ae1dce01413d1f8eb0614fd65d831e26e649dc1a31699b7275455c57aa563b59" + "sha256:03735b242439f8dbedc0f33ac9d0e980d755d19ed5e51dda1dcd866d9422edc8" ], "index": "pypi", - "version": "==0.3.0" + "version": "==0.3.1" }, "python-dateutil": { "hashes": [ @@ -829,19 +829,19 @@ }, "pyyaml": { "hashes": [ - "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", - "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", - "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", - "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", - "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", - "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", - "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", - "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", - "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", - "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", - "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], - "version": "==5.2" + "version": "==5.3" }, "pyzbar": { "hashes": [ @@ -928,10 +928,10 @@ }, "shodan": { "hashes": [ - "sha256:eab999bca9d3b30e6fc549e609194ff2d6fac3caea252414e1d8d735efab8342" + "sha256:ed3c38c749a5d77c935b226b6a7761e972269bd0d55c5c08526af73896aa6edd" ], "index": "pypi", - "version": "==1.21.0" + "version": "==1.21.2" }, "sigmatools": { "hashes": [ @@ -963,12 +963,12 @@ }, "sparqlwrapper": { "hashes": [ - "sha256:14ec551f0d60b4a496ffcc31f15337e844c085b8ead8cbe9a7178748a6de3794", - "sha256:21928e7a97f565e772cdeeb0abad428960f4307e3a13dbdd8f6d3da8a6a506c9", - "sha256:abc3e7eadcad32fa69a85c003853e2f6f73bda6cc999853838f401a5a1ea1109" + "sha256:357ee8a27bc910ea13d77836dbddd0b914991495b8cc1bf70676578155e962a8", + "sha256:c7f9c9d8ebb13428771bc3b6dee54197422507dcc3dea34e30d5dcfc53478dec", + "sha256:d6a66b5b8cda141660e07aeb00472db077a98d22cb588c973209c7336850fb3c" ], "index": "pypi", - "version": "==1.8.4" + "version": "==1.8.5" }, "stix2-patterns": { "hashes": [ @@ -1028,14 +1028,22 @@ ], "version": "==0.14.0" }, - "vulners": { + "vt-graph-api": { "hashes": [ - "sha256:245c07e49e55a604efde43cba723ac7b9345247e5ac8c4f998dcd36c05e4b1b9", - "sha256:82d47d7de208289a746bdb2dd9daf0fadf9fd290618015126091c7d9e2f8a96c", - "sha256:ef0c8e8c4e7d75fbd4d5bb1195109bd7a5b142f60dddc6cea77b3e20a3de1fa8" + "sha256:200c4f5a7c0a518502e890c4f4508a5ea042af9407d2889ef16a17ef11b7d25c", + "sha256:223c1cf32d69e10b5d3e178ec315589c7dfa7d43ccff6630a11ed5c5f498715c" ], "index": "pypi", - "version": "==1.5.4" + "version": "==1.0.1" + }, + "vulners": { + "hashes": [ + "sha256:00ff8744d07f398880afc1efcab6dac4abb614c84553fa31b2d439f986b8e0db", + "sha256:90a855915b4fb4dbd0325643d9e643602975fcb931162e5dc2e7778d1daa2fd8", + "sha256:f230bfcd42663326b7c9b8fa117752e26cad4ccca528caaab531c5b592af8cb5" + ], + "index": "pypi", + "version": "==1.5.5" }, "wand": { "hashes": [ @@ -1047,10 +1055,10 @@ }, "websocket-client": { "hashes": [ - "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", - "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" + "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", + "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" ], - "version": "==0.56.0" + "version": "==0.57.0" }, "wrapt": { "hashes": [ @@ -1068,10 +1076,10 @@ }, "xlsxwriter": { "hashes": [ - "sha256:027fa3d22ccfb5da5d77c29ed740aece286a9a6cc101b564f2f7ca11eb1d490b", - "sha256:5d480cee5babf3865227d5c81269d96be8e87914fc96403ca6fa1b1e4f64c080" + "sha256:18fe8f891a4adf7556c05d56059e136f9fbce5b19f9335f6d7b42c389c4592bc", + "sha256:5d3630ff9b2a277c939bd5053d0e7466499593abebbab9ce1dc9b1481a8ebbb6" ], - "version": "==1.2.6" + "version": "==1.2.7" }, "yara-python": { "hashes": [ @@ -1152,39 +1160,39 @@ }, "coverage": { "hashes": [ - "sha256:0cd13a6e98c37b510a2d34c8281d5e1a226aaf9b65b7d770ef03c63169965351", - "sha256:1a4b6b6a2a3a6612e6361130c2cc3dc4378d8c221752b96167ccbad94b47f3cd", - "sha256:2ee55e6dba516ddf6f484aa83ccabbb0adf45a18892204c23486938d12258cde", - "sha256:3be5338a2eb4ef03c57f20917e1d12a1fd10e3853fed060b6d6b677cb3745898", - "sha256:44b783b02db03c4777d8cf71bae19eadc171a6f2a96777d916b2c30a1eb3d070", - "sha256:475bf7c4252af0a56e1abba9606f1e54127cdf122063095c75ab04f6f99cf45e", - "sha256:47c81ee687eafc2f1db7f03fbe99aab81330565ebc62fb3b61edfc2216a550c8", - "sha256:4a7f8e72b18f2aca288ff02255ce32cc830bc04d993efbc87abf6beddc9e56c0", - "sha256:50197163a22fd17f79086e087a787883b3ec9280a509807daf158dfc2a7ded02", - "sha256:56b13000acf891f700f5067512b804d1ec8c301d627486c678b903859d07f798", - "sha256:79388ae29c896299b3567965dbcd93255f175c17c6c7bca38614d12718c47466", - "sha256:79fd5d3d62238c4f583b75d48d53cdae759fe04d4fb18fe8b371d88ad2b6f8be", - "sha256:7fe3e2fde2bf1d7ce25ebcd2d3de3650b8d60d9a73ce6dcef36e20191291613d", - "sha256:81042a24f67b96e4287774014fa27220d8a4d91af1043389e4d73892efc89ac6", - "sha256:81326f1095c53111f8afc95da281e1414185f4a538609a77ca50bdfa39a6c207", - "sha256:8873dc0d8f42142ea9f20c27bbdc485190fff93823c6795be661703369e5877d", - "sha256:88d2cbcb0a112f47eef71eb95460b6995da18e6f8ca50c264585abc2c473154b", - "sha256:91f2491aeab9599956c45a77c5666d323efdec790bfe23fcceafcd91105d585a", - "sha256:979daa8655ae5a51e8e7a24e7d34e250ae8309fd9719490df92cbb2fe2b0422b", - "sha256:9c871b006c878a890c6e44a5b2f3c6291335324b298c904dc0402ee92ee1f0be", - "sha256:a6d092545e5af53e960465f652e00efbf5357adad177b2630d63978d85e46a72", - "sha256:b5ed7837b923d1d71c4f587ae1539ccd96bfd6be9788f507dbe94dab5febbb5d", - "sha256:ba259f68250f16d2444cbbfaddaa0bb20e1560a4fdaad50bece25c199e6af864", - "sha256:be1d89614c6b6c36d7578496dc8625123bda2ff44f224cf8b1c45b810ee7383f", - "sha256:c1b030a79749aa8d1f1486885040114ee56933b15ccfc90049ba266e4aa2139f", - "sha256:c95bb147fab76f2ecde332d972d8f4138b8f2daee6c466af4ff3b4f29bd4c19e", - "sha256:d52c1c2d7e856cecc05aa0526453cb14574f821b7f413cc279b9514750d795c1", - "sha256:d609a6d564ad3d327e9509846c2c47f170456344521462b469e5cb39e48ba31c", - "sha256:e1bad043c12fb58e8c7d92b3d7f2f49977dcb80a08a6d1e7a5114a11bf819fca", - "sha256:e5a675f6829c53c87d79117a8eb656cc4a5f8918185a32fc93ba09778e90f6db", - "sha256:fec32646b98baf4a22fdceb08703965bd16dea09051fbeb31a04b5b6e72b846c" + "sha256:189aac76d6e0d7af15572c51892e7326ee451c076c5a50a9d266406cd6c49708", + "sha256:1bf7ba2af1d373a1750888724f84cffdfc697738f29a353c98195f98fc011509", + "sha256:1f4ee8e2e4243971618bc16fcc4478317405205f135e95226c2496e2a3b8dbbf", + "sha256:225e79a5d485bc1642cb7ba02281419c633c216cdc6b26c26494ba959f09e69f", + "sha256:23688ff75adfa8bfa2a67254d889f9bdf9302c27241d746e17547c42c732d3f4", + "sha256:28f7f73b34a05e23758e860a89a7f649b85c6749e252eff60ebb05532d180e86", + "sha256:2d0cb9b1fe6ad0d915d45ad3d87f03a38e979093a98597e755930db1f897afae", + "sha256:47874b4711c5aeb295c31b228a758ce3d096be83dc37bd56da48ed99efb8813b", + "sha256:511ec0c00840e12fb4e852e4db58fa6a01ca4da72f36a9766fae344c3d502033", + "sha256:53e7438fef0c97bc248f88ba1edd10268cd94d5609970aaf87abbe493691af87", + "sha256:569f9ee3025682afda6e9b0f5bb14897c0db03f1a1dc088b083dd36e743f92bb", + "sha256:593853aa1ac6dcc6405324d877544c596c9d948ef20d2e9512a0f5d2d3202356", + "sha256:5b0a07158360d22492f9abd02a0f2ee7981b33f0646bf796598b7673f6bbab14", + "sha256:7ca3db38a61f3655a2613ee2c190d63639215a7a736d3c64cc7bbdb002ce6310", + "sha256:7d1cc7acc9ce55179616cf72154f9e648136ea55987edf84addbcd9886ffeba2", + "sha256:88b51153657612aea68fa684a5b88037597925260392b7bb4509d4f9b0bdd889", + "sha256:955ec084f549128fa2702f0b2dc696392001d986b71acd8fd47424f28289a9c3", + "sha256:b251c7092cbb6d789d62dc9c9e7c4fb448c9138b51285c36aeb72462cad3600e", + "sha256:bd82b684bb498c60ef47bb1541a50e6d006dde8579934dcbdbc61d67d1ea70d9", + "sha256:bfe102659e2ec13b86c7f3b1db6c9a4e7beea4255058d006351339e6b342d5d2", + "sha256:c1e4e39e43057396a5e9d069bfbb6ffeee892e40c5d2effbd8cd71f34ee66c4d", + "sha256:cb2b74c123f65e8166f7e1265829a6c8ed755c3cd16d7f50e75a83456a5f3fd7", + "sha256:cca38ded59105f7705ef6ffe1e960b8db6c7d8279c1e71654a4775ab4454ca15", + "sha256:cf908840896f7aa62d0ec693beb53264b154f972eb8226fb864ac38975590c4f", + "sha256:d095a7b473f8a95f7efe821f92058c8a2ecfb18f8db6677ae3819e15dc11aaae", + "sha256:d22b4297e7e4225ccf01f1aa55e7a96412ea0796b532dd614c3fcbafa341128e", + "sha256:d4a2b578a7a70e0c71f662705262f87a456f1e6c1e40ada7ea699abaf070a76d", + "sha256:ddeb42a3d5419434742bf4cc71c9eaa22df3b76808e23a82bd0b0bd360f1a9f1", + "sha256:e65a5aa1670db6263f19fdc03daee1d7dbbadb5cb67fd0a1f16033659db13c1d", + "sha256:eaad65bd20955131bcdb3967a4dea66b4e4d4ca488efed7c00d91ee0173387e8", + "sha256:f45fba420b94165c17896861bb0e8b27fb7abdcedfeb154895d8553df90b7b00" ], - "version": "==5.0" + "version": "==5.0.2" }, "entrypoints": { "hashes": [ @@ -1241,10 +1249,10 @@ }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pluggy": { "hashes": [ @@ -1255,10 +1263,10 @@ }, "py": { "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", + "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" ], - "version": "==1.8.0" + "version": "==1.8.1" }, "pycodestyle": { "hashes": [ @@ -1276,10 +1284,10 @@ }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "pytest": { "hashes": [ @@ -1316,10 +1324,10 @@ }, "wcwidth": { "hashes": [ - "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", - "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", + "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" ], - "version": "==0.1.7" + "version": "==0.1.8" }, "zipp": { "hashes": [ From f5452055f607fcdba8e407a78fa106ff172c1155 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 10 Jan 2020 10:31:52 +0100 Subject: [PATCH 09/50] fix: Fixed vt_graph imports --- misp_modules/lib/__init__.py | 2 ++ misp_modules/lib/vt_graph_parser/__init__.py | 8 ++------ misp_modules/lib/vt_graph_parser/helpers/__init__.py | 3 +++ misp_modules/lib/vt_graph_parser/helpers/parsers.py | 2 +- misp_modules/lib/vt_graph_parser/importers/__init__.py | 7 +------ misp_modules/lib/vt_graph_parser/importers/base.py | 2 +- .../lib/vt_graph_parser/importers/pymisp_response.py | 4 ++-- misp_modules/modules/export_mod/vt_graph.py | 2 +- 8 files changed, 13 insertions(+), 17 deletions(-) diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index 57a2505..c078cf7 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1 +1,3 @@ +from .vt_graph_parser import * # noqa + all = ['joe_parser', 'lastline_api'] diff --git a/misp_modules/lib/vt_graph_parser/__init__.py b/misp_modules/lib/vt_graph_parser/__init__.py index 2a4d339..abc02c5 100644 --- a/misp_modules/lib/vt_graph_parser/__init__.py +++ b/misp_modules/lib/vt_graph_parser/__init__.py @@ -4,9 +4,5 @@ This module provides methods to import graph from misp. """ -from lib.vt_graph_parser.importers import from_pymisp_response - - -__all__ = [ - "from_pymisp_response" -] +from .helpers import * # noqa +from .importers import * # noqa diff --git a/misp_modules/lib/vt_graph_parser/helpers/__init__.py b/misp_modules/lib/vt_graph_parser/helpers/__init__.py index 336faee..7e0ec86 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/__init__.py +++ b/misp_modules/lib/vt_graph_parser/helpers/__init__.py @@ -2,3 +2,6 @@ This modules provides functions and attributes to help MISP importers. """ + + +all = ["parsers", "rules", "wrappers"] diff --git a/misp_modules/lib/vt_graph_parser/helpers/parsers.py b/misp_modules/lib/vt_graph_parser/helpers/parsers.py index c621595..8ca5745 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/parsers.py +++ b/misp_modules/lib/vt_graph_parser/helpers/parsers.py @@ -4,7 +4,7 @@ This module provides parsers for MISP inputs. """ -from lib.vt_graph_parser.helpers.wrappers import MispAttribute +from vt_graph_parser.helpers.wrappers import MispAttribute MISP_INPUT_ATTR = [ diff --git a/misp_modules/lib/vt_graph_parser/importers/__init__.py b/misp_modules/lib/vt_graph_parser/importers/__init__.py index 129d870..c59197c 100644 --- a/misp_modules/lib/vt_graph_parser/importers/__init__.py +++ b/misp_modules/lib/vt_graph_parser/importers/__init__.py @@ -4,9 +4,4 @@ This module provides methods to import graphs from MISP. """ -from lib.vt_graph_parser.importers.pymisp_response import from_pymisp_response - - -__all__ = [ - "from_pymisp_response" -] +__all__ = ["base", "pymisp_response"] diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py index 4d9b855..ed5c0fc 100644 --- a/misp_modules/lib/vt_graph_parser/importers/base.py +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -5,7 +5,7 @@ This module provides a common method to import graph from misp attributes. import vt_graph_api -from lib.vt_graph_parser.helpers.rules import MispEventInitialRule +from vt_graph_parser.helpers.rules import MispEventInitialRule def import_misp_graph( diff --git a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py index 86a3b25..e0e834b 100644 --- a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +++ b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py @@ -5,8 +5,8 @@ response payload giving by MISP API directly. """ -from lib.vt_graph_parser.helpers.parsers import parse_pymisp_response -from lib.vt_graph_parser.importers.base import import_misp_graph +from vt_graph_parser.helpers.parsers import parse_pymisp_response +from vt_graph_parser.importers.base import import_misp_graph def from_pymisp_response( diff --git a/misp_modules/modules/export_mod/vt_graph.py b/misp_modules/modules/export_mod/vt_graph.py index d8b3359..70c1952 100644 --- a/misp_modules/modules/export_mod/vt_graph.py +++ b/misp_modules/modules/export_mod/vt_graph.py @@ -3,7 +3,7 @@ import base64 import json -from lib.vt_graph_parser import from_pymisp_response +from vt_graph_parser.importers.pymisp_response import from_pymisp_response misperrors = { From 35c438e6ee536dd51e5462e3b82d21a962571989 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 10 Jan 2020 10:38:12 +0100 Subject: [PATCH 10/50] fix: typo --- misp_modules/lib/vt_graph_parser/helpers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/lib/vt_graph_parser/helpers/__init__.py b/misp_modules/lib/vt_graph_parser/helpers/__init__.py index 7e0ec86..8f9f660 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/__init__.py +++ b/misp_modules/lib/vt_graph_parser/helpers/__init__.py @@ -4,4 +4,4 @@ This modules provides functions and attributes to help MISP importers. """ -all = ["parsers", "rules", "wrappers"] +__all__ = ["parsers", "rules", "wrappers"] From b3bc533bc3d9608b1d56d764fb3c7a05c1ffc17b Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 10 Jan 2020 15:02:59 +0100 Subject: [PATCH 11/50] chg: Making ipasn module return asn object(s) - Latest changes on the returned value as string broke the freetext parser, because no asn number could be parsed when we return the full json blob as a freetext attribute - Now returning asn object(s) with a reference to the initial attribute --- misp_modules/modules/expansion/ipasn.py | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index cfdbaf5..ca10a7a 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -2,22 +2,40 @@ import json from pyipasnhistory import IPASNHistory +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'format': 'misp_standard'} moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)', 'module-type': ['expansion', 'hover']} +def parse_result(attribute, values): + event = MISPEvent() + initial_attribute = MISPAttribute() + initial_attribute.from_dict(**attribute) + event.add_attribute(**initial_attribute) + mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')} + print(values) + for last_seen, response in values['response'].items(): + asn = MISPObject('asn') + asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen}) + for feature, attribute_fields in mapping.items(): + attribute_type, object_relation = attribute_fields + asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]}) + asn.add_reference(initial_attribute.uuid, 'related-to') + event.add_object(**asn) + event = json.loads(event.to_json()) + return {key: event[key] for key in ('Attribute', 'Object')} + + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] + if request.get('attribute') and request['attribute'].get('type') in mispattributes['input']: + toquery = request['attribute']['value'] else: misperrors['error'] = "Unsupported attributes type" return misperrors @@ -28,7 +46,7 @@ def handler(q=False): if not values: misperrors['error'] = 'Unable to find the history of this IP' return misperrors - return {'results': [{'types': mispattributes['output'], 'values': [str(values)]}]} + return {'results': parse_result(request['attribute'], values)} def introspection(): From 8db9891c838ba49e7709653a849a5c82344e2ce9 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 10 Jan 2020 15:12:52 +0100 Subject: [PATCH 12/50] fix: Updated ipasn test following the latest changes on the module --- tests/test_expansions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 93ee69d..c6d5944 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -237,9 +237,7 @@ class TestExpansions(unittest.TestCase): def test_ipasn(self): query = {"module": "ipasn", "ip-dst": "1.1.1.1"} response = self.misp_modules_post(query) - key = list(self.get_values(response)['response'].keys())[0] - entry = self.get_values(response)['response'][key]['asn'] - self.assertEqual(entry, '13335') + self.assertEqual(self.get_object(response), 'asn') def test_macaddess_io(self): module_name = 'macaddress_io' From 31a74a10c1815245f8cc2c245e52840127301e2e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 10 Jan 2020 15:37:54 +0100 Subject: [PATCH 13/50] fix: Fixed ipasn test input format + module version updated --- misp_modules/modules/expansion/ipasn.py | 2 +- tests/test_expansions.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index ca10a7a..3c6867c 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -6,7 +6,7 @@ from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst'], 'format': 'misp_standard'} -moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', +moduleinfo = {'version': '0.2', 'author': 'Raphaël Vinot', 'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)', 'module-type': ['expansion', 'hover']} diff --git a/tests/test_expansions.py b/tests/test_expansions.py index c6d5944..879c590 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -235,7 +235,10 @@ class TestExpansions(unittest.TestCase): self.assertTrue(value.startswith('{"ip":"1.1.1.1","status":"ok"')) def test_ipasn(self): - query = {"module": "ipasn", "ip-dst": "1.1.1.1"} + query = {"module": "ipasn", + "attribute": {"type": "ip-src", + "value": "149.13.33.14", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} response = self.misp_modules_post(query) self.assertEqual(self.get_object(response), 'asn') From a88f19942f75885b74678314c710925d1eafa29a Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 10 Jan 2020 16:19:00 +0100 Subject: [PATCH 14/50] new: Updated ipasn and added vt_graph documentation --- README.md | 37 ++++++++++++++++++------------------ doc/README.md | 24 +++++++++++++++++++++-- doc/expansion/ipasn.json | 4 ++-- doc/export_mod/vt_graph.json | 9 +++++++++ 4 files changed, 52 insertions(+), 22 deletions(-) create mode 100644 doc/export_mod/vt_graph.json diff --git a/README.md b/README.md index d0296a8..af78ca5 100644 --- a/README.md +++ b/README.md @@ -89,27 +89,28 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj ### Export modules -* [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). -* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. -* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). -* [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. -* [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. -* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query. -* [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. -* [osquery](misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. -* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. -* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. +* [CEF](misp_modules/modules/export_mod/cef_export.py) - module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) - module to export as rule for the Cisco FireSight manager ACL. +* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) - module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). +* [Lite Export](misp_modules/modules/export_mod/liteexport.py) - module to export a lite event. +* [PDF export](misp_modules/modules/export_mod/pdfexport.py) - module to export an event in PDF. +* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) - module to export applicable attributes from an event to a mass EQL query. +* [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) - module to export in Nexthink query format. +* [osquery](misp_modules/modules/export_mod/osqueryexport.py) - module to export in [osquery](https://osquery.io/) query format. +* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) - module to export in ThreatConnect CSV format. +* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) - module to export in ThreatStream format. +* [VirusTotal Graph](misp_modules/modules/export_mod/vt_graph.py) - Module to create a VirusTotal graph out of an event. ### Import modules -* [CSV import](misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. -* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. -* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. -* [GoAML import](misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. -* [Joe Sandbox import](misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report. -* [Lastline import](misp_modules/modules/import_mod/lastline_import.py) Module to import Lastline analysis reports. -* [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. -* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. +* [CSV import](misp_modules/modules/import_mod/csvimport.py) - Customizable CSV import module. +* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) - Cuckoo JSON import. +* [Email Import](misp_modules/modules/import_mod/email_import.py) - Email import module for MISP to import basic metadata. +* [GoAML import](misp_modules/modules/import_mod/goamlimport.py) - Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. +* [Joe Sandbox import](misp_modules/modules/import_mod/joe_import.py) - Parse data from a Joe Sandbox json report. +* [Lastline import](misp_modules/modules/import_mod/lastline_import.py) - Module to import Lastline analysis reports. +* [OCR](misp_modules/modules/import_mod/ocr.py) - Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) - OpenIOC import based on PyMISP library. * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. diff --git a/doc/README.md b/doc/README.md index 64df950..2049803 100644 --- a/doc/README.md +++ b/doc/README.md @@ -532,11 +532,11 @@ Module to access intelmqs eventdb. Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). - **features**: ->This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input. +>This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object. - **input**: >An IP address MISP attribute. - **output**: ->Text describing additional information about the input after a query on the IPASN-history database. +>Asn object(s) objects related to the IP address used as input. - **references**: >https://github.com/D4-project/IPASN-History - **requirements**: @@ -1586,6 +1586,26 @@ Module to export a structured CSV file for uploading to ThreatConnect. ----- +#### [vt_graph](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/vt_graph.py) + + + +This module is used to create a VirusTotal Graph from a MISP event. +- **features**: +>The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event. +> +>Once the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal. +- **input**: +>A MISP event. +- **output**: +>Link of the VirusTotal Graph created for the event. +- **references**: +>https://www.virustotal.com/gui/graph-overview +- **requirements**: +>vt_graph_api, the python library to query the VirusTotal graph API + +----- + ## Import Modules #### [csvimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) diff --git a/doc/expansion/ipasn.json b/doc/expansion/ipasn.json index 68b10d1..8caed92 100644 --- a/doc/expansion/ipasn.json +++ b/doc/expansion/ipasn.json @@ -2,7 +2,7 @@ "description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).", "requirements": ["pyipasnhistory: Python library to access IPASN-history instance"], "input": "An IP address MISP attribute.", - "output": "Text describing additional information about the input after a query on the IPASN-history database.", + "output": "Asn object(s) objects related to the IP address used as input.", "references": ["https://github.com/D4-project/IPASN-History"], - "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input." + "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object." } diff --git a/doc/export_mod/vt_graph.json b/doc/export_mod/vt_graph.json new file mode 100644 index 0000000..e317730 --- /dev/null +++ b/doc/export_mod/vt_graph.json @@ -0,0 +1,9 @@ +{ + "description": "This module is used to create a VirusTotal Graph from a MISP event.", + "logo": "logos/virustotal.png", + "requirements": ["vt_graph_api, the python library to query the VirusTotal graph API"], + "features": "The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal.", + "references": ["https://www.virustotal.com/gui/graph-overview"], + "input": "A MISP event.", + "output": "Link of the VirusTotal Graph created for the event." +} From 610c99ce7b9bb6b034d2f3e50e247229586be263 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Fri, 17 Jan 2020 10:58:31 +0100 Subject: [PATCH 15/50] Fix error message in Public VT module --- misp_modules/modules/expansion/virustotal_public.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index 69c2c85..a6c093a 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -85,8 +85,9 @@ class DomainQuery(VirusTotalParser): whois_object = MISPObject(whois) whois_object.add_attribute('text', type='text', value=query_result[whois]) self.misp_event.add_object(**whois_object) - siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) - self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) + if 'domain_siblings' in query_result['domain_siblings']: + siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) + self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) self.parse_urls(query_result) def parse_siblings(self, domain): From 036933ea14d4f90bd7c05e2466837d0c8c107ab7 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Fri, 17 Jan 2020 11:26:35 +0100 Subject: [PATCH 16/50] 2nd fix for VT Public module --- misp_modules/modules/expansion/virustotal_public.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index a6c093a..f31855e 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -85,9 +85,10 @@ class DomainQuery(VirusTotalParser): whois_object = MISPObject(whois) whois_object.add_attribute('text', type='text', value=query_result[whois]) self.misp_event.add_object(**whois_object) - if 'domain_siblings' in query_result['domain_siblings']: - siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) - self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) + if 'domain_siblings' in query_result: + siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) + if 'subdomains' in query_result: + self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) self.parse_urls(query_result) def parse_siblings(self, domain): From 66bf650b79e8a1c5d094a0af79fae5f2ac9aae40 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Sat, 28 Dec 2019 15:57:15 +0100 Subject: [PATCH 17/50] change: migrate to analysis API when submitting tasks to Lastline --- doc/README.md | 4 +- doc/expansion/lastline_query.json | 2 +- doc/expansion/lastline_submit.json | 2 +- doc/import_mod/lastline_import.json | 2 +- misp_modules/lib/lastline_api.py | 1046 ++++++++++------- .../modules/expansion/lastline_query.py | 13 +- .../modules/expansion/lastline_submit.py | 30 +- .../modules/import_mod/lastline_import.py | 13 +- 8 files changed, 638 insertions(+), 474 deletions(-) diff --git a/doc/README.md b/doc/README.md index 2049803..7e6bee3 100644 --- a/doc/README.md +++ b/doc/README.md @@ -613,6 +613,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re Query Lastline with an analysis link and parse the report into MISP attributes and objects. The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module. - **features**: +>The module requires a Lastline Portal `username` and `password`. >The module uses the new format and it is able to return MISP attributes and objects. >The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module. - **input**: @@ -630,7 +631,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit] Module to submit a file or URL to Lastline. - **features**: ->The module requires a Lastline API key and token (or username and password). +>The module requires a Lastline Analysis `api_token` and `key`. >When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module. - **input**: >File or URL to submit to Lastline. @@ -1701,6 +1702,7 @@ A module to import data from a Joe Sandbox analysis json report. Module to import and parse reports from Lastline analysis links. - **features**: +>The module requires a Lastline Portal `username` and `password`. >The module uses the new format and it is able to return MISP attributes and objects. >The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module. - **input**: diff --git a/doc/expansion/lastline_query.json b/doc/expansion/lastline_query.json index 0d5da39..6165890 100644 --- a/doc/expansion/lastline_query.json +++ b/doc/expansion/lastline_query.json @@ -5,5 +5,5 @@ "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", "references": ["https://www.lastline.com"], - "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." } diff --git a/doc/expansion/lastline_submit.json b/doc/expansion/lastline_submit.json index cf5ef57..d053f55 100644 --- a/doc/expansion/lastline_submit.json +++ b/doc/expansion/lastline_submit.json @@ -5,5 +5,5 @@ "input": "File or URL to submit to Lastline.", "output": "Link to the report generated by Lastline.", "references": ["https://www.lastline.com"], - "features": "The module requires a Lastline API key and token (or username and password).\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." + "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." } diff --git a/doc/import_mod/lastline_import.json b/doc/import_mod/lastline_import.json index 1d4c15d..99414e0 100644 --- a/doc/import_mod/lastline_import.json +++ b/doc/import_mod/lastline_import.json @@ -5,5 +5,5 @@ "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", "references": ["https://www.lastline.com"], - "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." } diff --git a/misp_modules/lib/lastline_api.py b/misp_modules/lib/lastline_api.py index a6912b8..83726ad 100644 --- a/misp_modules/lib/lastline_api.py +++ b/misp_modules/lib/lastline_api.py @@ -30,19 +30,21 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import ipaddress +import abc import logging +import io +import ipaddress +import pymisp import re import requests -import pymisp - from urllib import parse -DEFAULT_LASTLINE_API = "https://user.lastline.com/papi" +DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi" +DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com" -HOSTED_LASTLINE_DOMAINS = frozenset([ +LL_HOSTED_DOMAINS = frozenset([ "user.lastline.com", "user.emea.lastline.com", ]) @@ -53,61 +55,66 @@ def purge_none(d): return {k: v for k, v in d.items() if v is not None} -def get_analysis_link(api_url, uuid): +def get_task_link(uuid, analysis_url=None, portal_url=None): """ - Get the analysis link of a task given the task uuid. + Get the task link given the task uuid and at least one API url. - :param str api_url: the URL :param str uuid: the task uuid + :param str|None analysis_url: the URL to the analysis API endpoint + :param str|None portal_url: the URL to the portal API endpoint :rtype: str - :return: the analysis link + :return: the task link + :raises ValueError: if not enough parameters have been provided """ + if not analysis_url and not portal_url: + raise ValueError("Neither analysis URL or portal URL have been specified") + if analysis_url: + portal_url = "{}/papi".format(analysis_url.replace("analysis.", "user.")) portal_url_path = "../portal#/analyst/task/{}/overview".format(uuid) - analysis_link = parse.urljoin(api_url, portal_url_path) - return analysis_link + return parse.urljoin(portal_url, portal_url_path) -def get_uuid_from_link(analysis_link): +def get_portal_url_from_task_link(task_link): """ - Return task uuid from link or raise ValueError exception. + Return the portal API url related to the provided task link. - :param str analysis_link: a link + :param str task_link: a link + :rtype: str + :return: the portal API url + """ + parsed_uri = parse.urlparse(task_link) + return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) + + +def get_uuid_from_task_link(task_link): + """ + Return the uuid from a task link. + + :param str task_link: a link :rtype: str :return: the uuid :raises ValueError: if the link contains not task uuid """ try: - return re.findall("[a-fA-F0-9]{32}", analysis_link)[0] + return re.findall("[a-fA-F0-9]{32}", task_link)[0] except IndexError: raise ValueError("Link does not contain a valid task uuid") -def is_analysis_hosted(analysis_link): +def is_task_hosted(task_link): """ - Return whether the analysis link is pointing to a hosted submission. + Return whether the portal link is pointing to a hosted submission. - :param str analysis_link: a link + :param str task_link: a link :rtype: boolean - :return: whether the link is hosted + :return: whether the link points to a hosted analysis """ - for domain in HOSTED_LASTLINE_DOMAINS: - if domain in analysis_link: + for domain in LL_HOSTED_DOMAINS: + if domain in task_link: return True return False -def get_api_url_from_link(analysis_link): - """ - Return the API url related to the provided analysis link. - - :param str analysis_link: a link - :rtype: str - :return: the API url - """ - parsed_uri = parse.urlparse(analysis_link) - return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) - - class InvalidArgument(Exception): """Error raised invalid.""" @@ -135,6 +142,572 @@ class ApiError(Error): return "{}{}".format(self.error_msg, error_code) +class LastlineAbstractClient(abc.ABC): + """"A very basic HTTP client providing basic functionality.""" + + __metaclass__ = abc.ABCMeta + + SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login') + FORMATS = ["json", "xml"] + + @classmethod + def sanitize_login_params(cls, api_key, api_token, username, password): + """ + Return a dictionary with either API or USER credentials. + + :param str|None api_key: the API key + :param str|None api_token: the API token + :param str|None username: the username + :param str|None password: the password + :rtype: dict[str, str] + :return: the dictionary + :raises InvalidArgument: if too many values are invalid + """ + if api_key and api_token: + return { + "key": api_key, + "api_token": api_token, + } + elif username and password: + return { + "username": username, + "password": password, + } + else: + raise InvalidArgument("Arguments provided do not contain valid data") + + @classmethod + def get_login_params_from_dict(cls, d): + """ + Get the module configuration from a ConfigParser object. + + :param dict[str, str] d: the dictionary + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = d.get("key") + api_token = d.get("api_token") + username = d.get("username") + password = d.get("password") + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def get_login_params_from_conf(cls, conf, section_name): + """ + Get the module configuration from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = conf.get(section_name, "key", fallback=None) + api_token = conf.get(section_name, "api_token", fallback=None) + username = conf.get(section_name, "username", fallback=None) + password = conf.get(section_name, "password", fallback=None) + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def load_from_conf(cls, conf, section_name): + """ + Load client from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: T <- LastlineAbstractClient + :return: the loaded client + """ + url = conf.get(section_name, "url") + return cls(url, cls.get_login_params_from_conf(conf, section_name)) + + def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): + """ + Instantiate a Lastline mini client. + + :param str api_url: the URL of the API + :param dict[str, str]: the login parameters + :param int timeout: the timeout + :param boolean verify_ssl: whether to verify the SSL certificate + """ + self._url = api_url + self._login_params = login_params + self._timeout = timeout + self._verify_ssl = verify_ssl + self._session = None + self._logger = logging.getLogger(__name__) + + @abc.abstractmethod + def _login(self): + """Login using account-based or key-based methods.""" + + def _is_logged_in(self): + """Return whether we have an active session.""" + return self._session is not None + + @staticmethod + def _parse_response(response): + """ + Parse the response. + + :param requests.Response response: the response + :rtype: tuple(str|None, Error|ApiError) + :return: a tuple with mutually exclusive fields (either the response or the error) + """ + try: + ret = response.json() + if "success" not in ret: + return None, Error("no success field in response") + + if not ret["success"]: + error_msg = ret.get("error", "") + error_code = ret.get("error_code", None) + return None, ApiError(error_msg, error_code) + + if "data" not in ret: + return None, Error("no data field in response") + + return ret["data"], None + except ValueError as e: + return None, Error("Response not json {}".format(e)) + + def _handle_response(self, response, raw=False): + """ + Check a response for issues and parse the return. + + :param requests.Response response: the response + :param boolean raw: whether the raw body should be returned + :rtype: str + :return: if raw, return the response content; if not raw, the data field + :raises: CommunicationError, ApiError, Error + """ + # Check for HTTP errors, and re-raise in case + try: + response.raise_for_status() + except requests.RequestException as e: + _, err = self._parse_response(response) + if isinstance(err, ApiError): + err_msg = "{}: {}".format(e, err.error_msg) + else: + err_msg = "{}".format(e) + raise CommunicationError(err_msg) + + # Otherwise return the data (either parsed or not) but reraise if we have an API error + if raw: + return response.content + data, err = self._parse_response(response) + if err: + raise err + return data + + def _build_url(self, sub_api, parts, requested_format="json"): + if sub_api not in self.SUB_APIS: + raise InvalidArgument(sub_api) + if requested_format not in self.FORMATS: + raise InvalidArgument(requested_format) + num_parts = 2 + len(parts) + pattern = "/".join(["%s"] * num_parts) + ".%s" + params = [self._url, sub_api] + parts + [requested_format] + return pattern % tuple(params) + + def post(self, module, function, params=None, data=None, files=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="POST", + params=params, + data=data, + files=files, + fmt=fmt, + ) + + def get(self, module, function, params=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="GET", + params=params, + fmt=fmt, + ) + + def do_request( + self, + method, + url, + params=None, + data=None, + files=None, + fmt="json", + raw=False, + raw_response=False, + headers=None, + stream_response=False + ): + if raw_response: + raw = True + + if fmt: + fmt = fmt.lower().strip() + if fmt not in self.FORMATS: + raise InvalidArgument("Only json, xml, html and pdf supported") + elif not raw: + raise InvalidArgument("Unformatted response requires raw=True") + + if fmt != "json" and not raw: + raise InvalidArgument("Non-json format requires raw=True") + + if method not in ["POST", "GET"]: + raise InvalidArgument("Only POST and GET supported") + + if not self._is_logged_in(): + self._login() + + try: + try: + response = self._session.request( + method=method, + url=url, + data=data, + params=params, + files=files, + verify=self._verify_ssl, + timeout=self._timeout, + stream=stream_response, + headers=headers, + ) + except requests.RequestException as e: + raise CommunicationError(e) + + if raw_response: + return response + return self._handle_response(response, raw) + + except Error as e: + raise e + + except CommunicationError as e: + raise e + + +class AnalysisClient(LastlineAbstractClient): + + def _login(self): + """ + Creates auth session for malscape-service. + + Credentials are 'key' and 'api_token'. + """ + if self._session is None: + self._session = requests.session() + url = self._build_url("authentication", ["login"]) + self.do_request("POST", url, params=purge_none(self._login_params)) + + def get_progress(self, uuid): + """ + Get the completion progress of a given task. + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + url = self._build_url('analysis', ['get_progress']) + params = {'uuid': uuid} + return self.do_request("POST", url, params=params) + + def get_result(self, uuid): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + # better: use 'get_results()' but that would break + # backwards-compatibility + url = self._build_url('analysis', ['get']) + params = {'uuid': uuid} + return self.do_request("GET", url, params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + file_stream = io.BytesIO(file_data) + api_url = self._build_url("analysis", ["submit", "file"]) + params = purge_none({ + "bypass_cache": bypass_cache and 1 or None, + "analysis_timeout": analysis_timeout, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic and 1 or None, + "filename": file_name, + "password": password, + "full_report_score": -1, + }) + + files = purge_none({ + # If an explicit filename was provided, we can pass it down to + # python-requests to use it in the multipart/form-data. This avoids + # having python-requests trying to guess the filename based on stream + # attributes. + # + # The problem with this is that, if the filename is not ASCII, then + # this triggers a bug in flask/werkzeug which means the file is + # thrown away. Thus, we just force an ASCII name + "file": ('dummy-ascii-name-for-file-param', file_stream), + }) + + return self.do_request("POST", api_url, params=params, files=files) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + api_url = self._build_url("analysis", ["submit", "url"]) + params = purge_none({ + "url": url, + "referer": referer, + "bypass_cache": bypass_cache and 1 or None, + "user_agent": user_agent or None, + }) + return self.do_request("POST", api_url, params=params) + + +class PortalClient(LastlineAbstractClient): + + def _login(self): + """ + Login using account-based or key-based methods. + + Credentials are 'username' and 'password' + """ + if self._session is None: + self._session = requests.session() + self.post("login", function=None, data=self._login_params) + + def get_progress(self, uuid, analysis_instance=None): + """ + Get the completion progress of a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) + return self.get("analysis", "get_progress", params=params) + + def get_result(self, uuid, analysis_instance=None): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none( + { + "uuid": uuid, + "analysis_instance": analysis_instance, + "report_format": "json", + } + ) + return self.get("analysis", "get_result", params=params) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "url": url, + "bypass_cache": bypass_cache, + "referer": referer, + "user_agent": user_agent + } + ) + return self.post("analysis", "submit_url", params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "filename": file_name, + "password": password, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic, + "analysis_timeout": analysis_timeout, + "bypass_cache": bypass_cache, + } + ) + files = {"file": (file_name, file_data, "application/octet-stream")} + return self.post("analysis", "submit_file", params=params, files=files) + + class LastlineResultBaseParser(object): """ This is a parser to extract *basic* information from a Lastline result dictionary. @@ -247,7 +820,7 @@ class LastlineResultBaseParser(object): # Add sandbox info like score and sandbox type o = pymisp.MISPObject(name="sandbox-report") - sandbox_type = "saas" if is_analysis_hosted(analysis_link) else "on-premise" + sandbox_type = "saas" if is_task_hosted(analysis_link) else "on-premise" o.add_attribute("score", result["score"]) o.add_attribute("sandbox-type", sandbox_type) o.add_attribute("{}-sandbox".format(sandbox_type), "lastline") @@ -266,408 +839,3 @@ class LastlineResultBaseParser(object): # Add mitre techniques for technique in self._get_mitre_techniques(result): self.misp_event.add_tag(technique) - - -class LastlineCommunityHTTPClient(object): - """"A very basic HTTP client providing basic functionality.""" - - @classmethod - def sanitize_login_params(cls, api_key, api_token, username, password): - """ - Return a dictionary with either API or USER credentials. - - :param str|None api_key: the API key - :param str|None api_token: the API token - :param str|None username: the username - :param str|None password: the password - :rtype: dict[str, str] - :return: the dictionary - :raises InvalidArgument: if too many values are invalid - """ - if api_key and api_token: - return { - "api_key": api_key, - "api_token": api_token, - } - elif username and password: - return { - "username": username, - "password": password, - } - else: - raise InvalidArgument("Arguments provided do not contain valid data") - - @classmethod - def get_login_params_from_conf(cls, conf, section_name): - """ - Get the module configuration from a ConfigParser object. - - :param ConfigParser conf: the conf object - :param str section_name: the section name - :rtype: dict[str, str] - :return: the parsed configuration - """ - api_key = conf.get(section_name, "api_key", fallback=None) - api_token = conf.get(section_name, "api_token", fallback=None) - username = conf.get(section_name, "username", fallback=None) - password = conf.get(section_name, "password", fallback=None) - return cls.sanitize_login_params(api_key, api_token, username, password) - - @classmethod - def get_login_params_from_request(cls, request): - """ - Get the module configuration from a ConfigParser object. - - :param dict[str, any] request: the request object - :rtype: dict[str, str] - :return: the parsed configuration - """ - api_key = request.get("config", {}).get("api_key") - api_token = request.get("config", {}).get("api_token") - username = request.get("config", {}).get("username") - password = request.get("config", {}).get("password") - return cls.sanitize_login_params(api_key, api_token, username, password) - - def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): - """ - Instantiate a Lastline mini client. - - :param str api_url: the URL of the API - :param dict[str, str]: the login parameters - :param int timeout: the timeout - :param boolean verify_ssl: whether to verify the SSL certificate - """ - self.__url = api_url - self.__login_params = login_params - self.__timeout = timeout - self.__verify_ssl = verify_ssl - self.__session = None - self.__logger = logging.getLogger(__name__) - - def __login(self): - """Login using account-based or key-based methods.""" - if self.__session is None: - self.__session = requests.session() - - login_url = "/".join([self.__url, "login"]) - try: - response = self.__session.request( - method="POST", - url=login_url, - data=self.__login_params, - verify=self.__verify_ssl, - timeout=self.__timeout, - proxies=None, - ) - except requests.RequestException as e: - raise CommunicationError(e) - - self.__handle_response(response) - - def __is_logged_in(self): - """Return whether we have an active session.""" - return self.__session is not None - - @staticmethod - def __parse_response(response): - """ - Parse the response. - - :param requests.Response response: the response - :rtype: tuple(str|None, Error|ApiError) - :return: a tuple with mutually exclusive fields (either the response or the error) - """ - try: - ret = response.json() - if "success" not in ret: - return None, Error("no success field in response") - - if not ret["success"]: - error_msg = ret.get("error", "") - error_code = ret.get("error_code", None) - return None, ApiError(error_msg, error_code) - - if "data" not in ret: - return None, Error("no data field in response") - - return ret["data"], None - except ValueError as e: - return None, Error("Response not json {}".format(e)) - - def __handle_response(self, response, raw=False): - """ - Check a response for issues and parse the return. - - :param requests.Response response: the response - :param boolean raw: whether the raw body should be returned - :rtype: str - :return: if raw, return the response content; if not raw, the data field - :raises: CommunicationError, ApiError, Error - """ - # Check for HTTP errors, and re-raise in case - try: - response.raise_for_status() - except requests.RequestException as e: - _, err = self.__parse_response(response) - if isinstance(err, ApiError): - err_msg = "{}: {}".format(e, err.error_msg) - else: - err_msg = "{}".format(e) - raise CommunicationError(err_msg) - - # Otherwise return the data (either parsed or not) but reraise if we have an API error - if raw: - return response.content - data, err = self.__parse_response(response) - if err: - raise err - return data - - def do_request( - self, - method, - module, - function, - params=None, - data=None, - files=None, - url=None, - fmt="JSON", - raw=False, - raw_response=False, - headers=None, - stream_response=False - ): - if raw_response: - raw = True - - if fmt: - fmt = fmt.lower().strip() - if fmt not in ["json", "xml", "html", "pdf"]: - raise InvalidArgument("Only json, xml, html and pdf supported") - elif not raw: - raise InvalidArgument("Unformatted response requires raw=True") - - if fmt != "json" and not raw: - raise InvalidArgument("Non-json format requires raw=True") - - if method not in ["POST", "GET"]: - raise InvalidArgument("Only POST and GET supported") - - function = function.strip(" /") - if not function: - raise InvalidArgument("No function provided") - - # Login after we verified that all arguments are fine - if not self.__is_logged_in(): - self.__login() - - url_parts = [url or self.__url] - module = module.strip(" /") - if module: - url_parts.append(module) - if fmt: - function_part = "%s.%s" % (function, fmt) - else: - function_part = function - url_parts.append(function_part) - url = "/".join(url_parts) - - try: - try: - response = self.__session.request( - method=method, - url=url, - data=data, - params=params, - files=files, - verify=self.__verify_ssl, - timeout=self.__timeout, - stream=stream_response, - headers=headers, - ) - except requests.RequestException as e: - raise CommunicationError(e) - - if raw_response: - return response - return self.__handle_response(response, raw) - - except Error as e: - raise e - - except CommunicationError as e: - raise e - - -class LastlineCommunityAPIClient(object): - """"A very basic API client providing basic functionality.""" - - def __init__(self, api_url, login_params): - """ - Instantiate the API client. - - :param str api_url: the URL to the API server - :param dict[str, str] login_params: the login parameters - """ - self._client = LastlineCommunityHTTPClient(api_url, login_params) - self._logger = logging.getLogger(__name__) - - def _post(self, module, function, params=None, data=None, files=None, fmt="JSON"): - return self._client.do_request( - method="POST", - module=module, - function=function, - params=params, - data=data, - files=files, - fmt=fmt, - ) - - def _get(self, module, function, params=None, fmt="JSON"): - return self._client.do_request( - method="GET", - module=module, - function=function, - params=params, - fmt=fmt, - ) - - def get_progress(self, uuid, analysis_instance=None): - """ - Get the completion progress of a given task. - - :param str uuid: the unique identifier of the submitted task - :param str analysis_instance: if set, defines the analysis instance to query - :rtype: dict[str, int] - :return: a dictionary like the the following: - { - "completed": 1, - "progress": 100 - } - """ - params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) - return self._get("analysis", "get_progress", params=params) - - def get_result(self, uuid, analysis_instance=None): - """ - Get report results for a given task. - - :param str uuid: the unique identifier of the submitted task - :param str analysis_instance: if set, defines the analysis instance to query - :rtype: dict[str, any] - :return: a dictionary like the the following: - { - "completed": 1, - "progress": 100 - } - """ - params = purge_none( - { - "uuid": uuid, - "analysis_instance": analysis_instance, - "report_format": "json", - } - ) - return self._get("analysis", "get_result", params=params) - - def submit_url( - self, - url, - referer=None, - user_agent=None, - bypass_cache=False, - ): - """ - Upload an URL to be analyzed. - - :param str url: the url to analyze - :param str|None referer: the referer - :param str|None user_agent: the user agent - :param boolean bypass_cache: bypass_cache - :rtype: dict[str, any] - :return: a dictionary like the following if the analysis is already available: - { - "submission": "2019-11-17 09:33:23", - "child_tasks": [...], - "reports": [...], - "submission_timestamp": "2019-11-18 16:11:04", - "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", - "score": 0, - "analysis_subject": { - "url": "https://www.google.com" - }, - "last_submission_timestamp": "2019-11-18 16:11:04" - } - - OR the following if the analysis is still pending: - - { - "submission_timestamp": "2019-11-18 13:59:25", - "task_uuid": "f3c0ae115d51001017ff8da768fa6049", - } - """ - params = purge_none( - { - "url": url, - "bypass_cache": bypass_cache, - "referer": referer, - "user_agent": user_agent - } - ) - return self._post(module="analysis", function="submit_url", params=params) - - def submit_file( - self, - file_data, - file_name=None, - password=None, - analysis_env=None, - allow_network_traffic=True, - analysis_timeout=None, - bypass_cache=False, - ): - """ - Upload a file to be analyzed. - - :param bytes file_data: the data as a byte sequence - :param str|None file_name: if set, represents the name of the file to submit - :param str|None password: if set, use it to extract the sample - :param str|None analysis_env: if set, e.g windowsxp - :param boolean allow_network_traffic: if set to False, deny network connections - :param int|None analysis_timeout: if set limit the duration of the analysis - :param boolean bypass_cache: whether to re-process a file (requires special permissions) - :rtype: dict[str, any] - :return: a dictionary in the following form if the analysis is already available: - { - "submission": "2019-11-17 09:33:23", - "child_tasks": [...], - "reports": [...], - "submission_timestamp": "2019-11-18 16:11:04", - "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", - "score": 0, - "analysis_subject": { - "url": "https://www.google.com" - }, - "last_submission_timestamp": "2019-11-18 16:11:04" - } - - OR the following if the analysis is still pending: - - { - "submission_timestamp": "2019-11-18 13:59:25", - "task_uuid": "f3c0ae115d51001017ff8da768fa6049", - } - """ - params = purge_none( - { - "filename": file_name, - "password": password, - "analysis_env": analysis_env, - "allow_network_traffic": allow_network_traffic, - "analysis_timeout": analysis_timeout, - "bypass_cache": bypass_cache, - } - ) - files = {"file": (file_name, file_data, "application/octet-stream")} - return self._post(module="analysis", function="submit_file", params=params, files=files) diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py index 4019b92..9fdc9de 100644 --- a/misp_modules/modules/expansion/lastline_query.py +++ b/misp_modules/modules/expansion/lastline_query.py @@ -27,8 +27,6 @@ moduleinfo = { } moduleconfig = [ - "api_key", - "api_token", "username", "password", ] @@ -51,24 +49,25 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) analysis_link = request['attribute']['value'] # The API url changes based on the analysis link host name - api_url = lastline_api.get_api_url_from_link(analysis_link) + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - task_uuid = lastline_api.get_uuid_from_link(analysis_link) + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) except (KeyError, ValueError) as e: misperrors["error"] = "Error processing input parameters: {}".format(e) return misperrors # Make the API calls try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -108,7 +107,7 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { diff --git a/misp_modules/modules/expansion/lastline_submit.py b/misp_modules/modules/expansion/lastline_submit.py index 0ae475a..1572955 100644 --- a/misp_modules/modules/expansion/lastline_submit.py +++ b/misp_modules/modules/expansion/lastline_submit.py @@ -33,13 +33,9 @@ moduleinfo = { } moduleconfig = [ - "api_url", - "api_key", + "url", "api_token", - "username", - "password", - # Module options - "bypass_cache", + "key", ] @@ -75,31 +71,31 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) - api_url = request.get("config", {}).get("api_url", lastline_api.DEFAULT_LASTLINE_API) + config = request.get("config", {}) + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - bypass_cache = request.get("config", {}).get("bypass_cache", False) - call_args = {"bypass_cache": __str_to_bool(bypass_cache)} + call_args = {} if "url" in request: # URLs are text strings - api_method = lastline_api.LastlineCommunityAPIClient.submit_url + api_method = lastline_api.AnalysisClient.submit_url call_args["url"] = request.get("url") else: data = request.get("data") # Malware samples are zip-encrypted and then base64 encoded if "malware-sample" in request: - api_method = lastline_api.LastlineCommunityAPIClient.submit_file + api_method = lastline_api.AnalysisClient.submit_file call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD) call_args["file_name"] = request.get("malware-sample").split("|", 1)[0] call_args["password"] = DEFAULT_ZIP_PASSWORD # Attachments are just base64 encoded elif "attachment" in request: - api_method = lastline_api.LastlineCommunityAPIClient.submit_file + api_method = lastline_api.AnalysisClient.submit_file call_args["file_data"] = base64.b64decode(data) call_args["file_name"] = request.get("attachment") @@ -112,7 +108,7 @@ def handler(q=False): # Make the API call try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.AnalysisClient(api_url, auth_data) response = api_method(api_client, **call_args) task_uuid = response.get("task_uuid") if not task_uuid: @@ -127,7 +123,7 @@ def handler(q=False): return misperrors # Assemble and return - analysis_link = lastline_api.get_analysis_link(api_url, task_uuid) + analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url) return { "results": [ @@ -152,12 +148,12 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { "config": a, - "url": "https://www.google.com", + "url": "https://www.google.exe.com", } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py index ff26b93..ebf88d8 100644 --- a/misp_modules/modules/import_mod/lastline_import.py +++ b/misp_modules/modules/import_mod/lastline_import.py @@ -29,8 +29,6 @@ moduleinfo = { } moduleconfig = [ - "api_key", - "api_token", "username", "password", ] @@ -65,24 +63,25 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) analysis_link = request["config"]["analysis_link"] # The API url changes based on the analysis link host name - api_url = lastline_api.get_api_url_from_link(analysis_link) + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - task_uuid = lastline_api.get_uuid_from_link(analysis_link) + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) except (KeyError, ValueError) as e: misperrors["error"] = "Error processing input parameters: {}".format(e) return misperrors # Make the API calls try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -122,7 +121,7 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { From f28aaf07c433692249b43a94f7cf6de70723b9b5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 21 Jan 2020 22:04:08 +0100 Subject: [PATCH 18/50] fix: [tests] Fixed BGP raking module test --- tests/test_expansions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 879c590..ee3a906 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -99,7 +99,7 @@ class TestExpansions(unittest.TestCase): def test_bgpranking(self): query = {"module": "bgpranking", "AS": "13335"} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET - Cloudflare, Inc., US') + self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET, US') def test_btc_steroids(self): query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} From 04685ea63e4cf38934945a8f384d60b00bee9b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20Sch=C3=B6lly?= Date: Fri, 24 Jan 2020 14:51:10 +0100 Subject: [PATCH 19/50] joe: (1) allow users to disable PE object import (2) set 'to_ids' to False --- misp_modules/lib/joe_parser.py | 87 +++++++++++-------- .../modules/expansion/joesandbox_query.py | 28 ++++-- misp_modules/modules/import_mod/joe_import.py | 22 ++++- 3 files changed, 91 insertions(+), 46 deletions(-) diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py index 00aa868..22a4918 100644 --- a/misp_modules/lib/joe_parser.py +++ b/misp_modules/lib/joe_parser.py @@ -51,12 +51,15 @@ signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'), class JoeParser(): - def __init__(self): + def __init__(self, config): self.misp_event = MISPEvent() self.references = defaultdict(list) self.attributes = defaultdict(lambda: defaultdict(set)) self.process_references = {} + self.import_pe = config["import_pe"] + self.create_mitre_attack = config["mitre_attack"] + def parse_data(self, data): self.data = data if self.analysis_type() == "file": @@ -72,7 +75,9 @@ class JoeParser(): if self.attributes: self.handle_attributes() - self.parse_mitre_attack() + + if self.create_mitre_attack: + self.parse_mitre_attack() def build_references(self): for misp_object in self.misp_event.objects: @@ -97,12 +102,12 @@ class JoeParser(): file_object = MISPObject('file') for key, mapping in dropped_file_mapping.items(): attribute_type, object_relation = mapping - file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]}) + file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False}) if droppedfile['@malicious'] == 'true': - file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'}) + file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False}) for h in droppedfile['value']: hash_type = dropped_hash_mapping[h['@algo']] - file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']}) + file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False}) self.misp_event.add_object(**file_object) self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({ 'referenced_uuid': file_object.uuid, @@ -132,9 +137,12 @@ class JoeParser(): for object_relation, attribute in attributes.items(): network_connection_object.add_attribute(object_relation, **attribute) network_connection_object.add_attribute('first-packet-seen', - **{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))}) + **{'type': 'datetime', + 'value': min(tuple(min(timestamp) for timestamp in data.values())), + 'to_ids': False}) for protocol in data.keys(): - network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol}) + network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), + **{'type': 'text', 'value': protocol, 'to_ids': False}) self.misp_event.add_object(**network_connection_object) self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, relationship_type='initiates')) @@ -143,8 +151,8 @@ class JoeParser(): network_connection_object = MISPObject('network-connection') for object_relation, attribute in attributes.items(): network_connection_object.add_attribute(object_relation, **attribute) - network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)}) - network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol}) + network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False}) + network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False}) self.misp_event.add_object(**network_connection_object) self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, relationship_type='initiates')) @@ -154,7 +162,8 @@ class JoeParser(): if screenshotdata: screenshotdata = screenshotdata['interesting']['$'] attribute = {'type': 'attachment', 'value': 'screenshot.jpg', - 'data': screenshotdata, 'disable_correlation': True} + 'data': screenshotdata, 'disable_correlation': True, + 'to_ids': False} self.misp_event.add_attribute(**attribute) def parse_system_behavior(self): @@ -166,9 +175,9 @@ class JoeParser(): general = process['general'] process_object = MISPObject('process') for feature, relation in process_object_fields.items(): - process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]}) + process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False}) start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S') - process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time}) + process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False}) self.misp_event.add_object(**process_object) for field, to_call in process_activities.items(): if process.get(field): @@ -203,7 +212,7 @@ class JoeParser(): url_object = MISPObject("url") self.analysisinfo_uuid = url_object.uuid - url_object.add_attribute("url", generalinfo["target"]["url"]) + url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False) self.misp_event.add_object(**url_object) def parse_fileinfo(self): @@ -213,10 +222,10 @@ class JoeParser(): self.analysisinfo_uuid = file_object.uuid for field in file_object_fields: - file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]}) + file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False}) for field, mapping in file_object_mapping.items(): attribute_type, object_relation = mapping - file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]}) + file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False}) arch = self.data['generalinfo']['arch'] if arch in arch_type_mapping: to_call = arch_type_mapping[arch] @@ -234,9 +243,9 @@ class JoeParser(): attribute_type = 'text' for comment, permissions in permission_lists.items(): permission_object = MISPObject('android-permission') - permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment)) + permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False)) for permission in permissions: - permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission)) + permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False)) self.misp_event.add_object(**permission_object) self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid, relationship_type='grants')) @@ -255,24 +264,24 @@ class JoeParser(): if elf.get('type'): # Haven't seen anything but EXEC yet in the files I tested attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type'] - elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value)) + elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False)) for feature, relation in elf_object_mapping.items(): if elf.get(feature): - elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature])) + elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False)) sections_number = len(fileinfo['sections']['section']) - elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number}) + elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) self.misp_event.add_object(**elf_object) for section in fileinfo['sections']['section']: section_object = MISPObject('elf-section') for feature in ('name', 'type'): if section.get(feature): - section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature])) + section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False)) if section.get('size'): - section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16))) + section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False)) for flag in section['flagsdesc']: try: attribute_value = elf_section_flags_mapping[flag] - section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value)) + section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False)) except KeyError: print(f'Unknown elf section flag: {flag}') continue @@ -281,6 +290,8 @@ class JoeParser(): relationship_type=relationship)) def parse_pe(self, fileinfo, file_object): + if not self.import_pe: + return try: peinfo = fileinfo['pe'] except KeyError: @@ -292,8 +303,8 @@ class JoeParser(): self.misp_event.add_object(**file_object) for field, mapping in pe_object_fields.items(): attribute_type, object_relation = mapping - pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]}) - pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)}) + pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False}) + pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False}) program_name = fileinfo['filename'] if peinfo['versions']: for feature in peinfo['versions']['version']: @@ -301,18 +312,18 @@ class JoeParser(): if name == 'InternalName': program_name = feature['value'] if name in pe_object_mapping: - pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']}) + pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False}) sections_number = len(peinfo['sections']['section']) - pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number}) + pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) signatureinfo = peinfo['signature'] if signatureinfo['signed']: signerinfo_object = MISPObject('authenticode-signerinfo') pe_object.add_reference(signerinfo_object.uuid, 'signed-by') self.misp_event.add_object(**pe_object) - signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name}) + signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False}) for feature, mapping in signerinfo_object_mapping.items(): attribute_type, object_relation = mapping - signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]}) + signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False}) self.misp_event.add_object(**signerinfo_object) else: self.misp_event.add_object(**pe_object) @@ -327,7 +338,7 @@ class JoeParser(): for feature, mapping in pe_section_object_mapping.items(): if section.get(feature): attribute_type, object_relation = mapping - section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]}) + section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False}) return section_object def parse_network_interactions(self): @@ -339,13 +350,13 @@ class JoeParser(): for key, mapping in domain_object_mapping.items(): attribute_type, object_relation = mapping domain_object.add_attribute(object_relation, - **{'type': attribute_type, 'value': domain[key]}) + **{'type': attribute_type, 'value': domain[key], 'to_ids': False}) self.misp_event.add_object(**domain_object) reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts') self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) else: attribute = MISPAttribute() - attribute.from_dict(**{'type': 'domain', 'value': domain['@name']}) + attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False}) self.misp_event.add_attribute(**attribute) reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) @@ -353,7 +364,7 @@ class JoeParser(): if ipinfo: for ip in ipinfo['ip']: attribute = MISPAttribute() - attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']}) + attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False}) self.misp_event.add_attribute(**attribute) reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference) @@ -363,7 +374,7 @@ class JoeParser(): target_id = int(url['@targetid']) current_path = url['@currentpath'] attribute = MISPAttribute() - attribute_dict = {'type': 'url', 'value': url['@name']} + attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False} if target_id != -1 and current_path != 'unknown': self.references[self.process_references[(target_id, current_path)]].append({ 'referenced_uuid': attribute.uuid, @@ -384,8 +395,8 @@ class JoeParser(): registry_key = MISPObject('registry-key') for field, mapping in regkey_object_mapping.items(): attribute_type, object_relation = mapping - registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]}) - registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())}) + registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False}) + registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False}) self.misp_event.add_object(**registry_key) self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid, relationship_type=relationship)) @@ -398,7 +409,7 @@ class JoeParser(): def create_attribute(self, attribute_type, attribute_value): attribute = MISPAttribute() - attribute.from_dict(**{'type': attribute_type, 'value': attribute_value}) + attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False}) self.misp_event.add_attribute(**attribute) return attribute.uuid @@ -419,5 +430,5 @@ class JoeParser(): attributes = {} for field, value in zip(network_behavior_fields, connection): attribute_type, object_relation = network_connection_object_mapping[field] - attributes[object_relation] = {'type': attribute_type, 'value': value} + attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False} return attributes diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py index dce63ea..1ace259 100644 --- a/misp_modules/modules/expansion/joesandbox_query.py +++ b/misp_modules/modules/expansion/joesandbox_query.py @@ -4,12 +4,13 @@ import json from joe_parser import JoeParser misperrors = {'error': 'Error'} -mispattributes = {'input': ['link'], 'format': 'misp_standard'} -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +inputSource = ['link'] + +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Query Joe Sandbox API with a report URL to get the parsed data.', 'module-type': ['expansion']} -moduleconfig = ['apiurl', 'apikey'] +moduleconfig = ['apiurl', 'apikey', 'import_pe', 'import_mitre_attack'] def handler(q=False): @@ -18,6 +19,11 @@ def handler(q=False): request = json.loads(q) apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api' apikey = request['config'].get('apikey') + parser_config = { + "import_pe": request["config"].get('import_pe', "false") == "true", + "mitre_attack": request["config"].get('import_mitre_attack', "false") == "true", + } + if not apikey: return {'error': 'No API key provided'} @@ -41,7 +47,7 @@ def handler(q=False): analysis_webid = joe_info['most_relevant_analysis']['webid'] - joe_parser = JoeParser() + joe_parser = JoeParser(parser_config) joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1]) joe_parser.parse_data(joe_data['analysis']) joe_parser.finalize_results() @@ -50,7 +56,19 @@ def handler(q=False): def introspection(): - return mispattributes + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['input'] = inputSource + except NameError: + pass + modulesetup['format'] = 'misp_standard' + return modulesetup def version(): diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py index d1c4d19..fbe7385 100644 --- a/misp_modules/modules/import_mod/joe_import.py +++ b/misp_modules/modules/import_mod/joe_import.py @@ -4,10 +4,20 @@ import json from joe_parser import JoeParser misperrors = {'error': 'Error'} -userConfig = {} +userConfig = { + "Import PE": { + "type": "Boolean", + "message": "Import PE Information", + }, + "Mitre Att&ck" : { + "type": "Boolean", + "message": "Import Mitre Att&ck techniques", + }, +} + inputSource = ['file'] -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Import for Joe Sandbox JSON reports', 'module-type': ['import']} @@ -18,10 +28,16 @@ def handler(q=False): if q is False: return False q = json.loads(q) + config = { + "import_pe": bool(int(q["config"]["Import PE"])), + "mitre_attack": bool(int(q["config"]["Mitre Att&ck"])), + } + data = base64.b64decode(q.get('data')).decode('utf-8') if not data: return json.dumps({'success': 0}) - joe_parser = JoeParser() + + joe_parser = JoeParser(config) joe_parser.parse_data(json.loads(data)['analysis']) joe_parser.finalize_results() return {'results': joe_parser.results} From b2c8f79220855e41db11d28248b01a06de8ed8c5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 24 Jan 2020 15:17:35 +0100 Subject: [PATCH 20/50] fix: Making pep8 happy --- misp_modules/modules/import_mod/joe_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py index fbe7385..0753167 100644 --- a/misp_modules/modules/import_mod/joe_import.py +++ b/misp_modules/modules/import_mod/joe_import.py @@ -9,7 +9,7 @@ userConfig = { "type": "Boolean", "message": "Import PE Information", }, - "Mitre Att&ck" : { + "Mitre Att&ck": { "type": "Boolean", "message": "Import Mitre Att&ck techniques", }, From 8f9940200beccd11b64b8505106e312ade227f9b Mon Sep 17 00:00:00 2001 From: Hendrik Date: Mon, 27 Jan 2020 07:43:46 +0100 Subject: [PATCH 21/50] Lastline verify_ssl option Helps people with on-prem boxes --- misp_modules/modules/expansion/lastline_query.py | 3 ++- misp_modules/modules/import_mod/lastline_import.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py index 9fdc9de..4ce4e47 100644 --- a/misp_modules/modules/expansion/lastline_query.py +++ b/misp_modules/modules/expansion/lastline_query.py @@ -29,6 +29,7 @@ moduleinfo = { moduleconfig = [ "username", "password", + "verify_ssl", ] @@ -67,7 +68,7 @@ def handler(q=False): # Make the API calls try: - api_client = lastline_api.PortalClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py index ebf88d8..37f6249 100644 --- a/misp_modules/modules/import_mod/lastline_import.py +++ b/misp_modules/modules/import_mod/lastline_import.py @@ -31,6 +31,7 @@ moduleinfo = { moduleconfig = [ "username", "password", + "verify_ssl", ] @@ -81,7 +82,7 @@ def handler(q=False): # Make the API calls try: - api_client = lastline_api.PortalClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") From acdc4b9d030772f92b2e4c2705b3a977c7a9da77 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 7 Feb 2020 12:20:12 +0100 Subject: [PATCH 22/50] fix: [VT] Disable SHA512 query for VT --- misp_modules/modules/expansion/virustotal.py | 7 +++---- misp_modules/modules/expansion/virustotal_public.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 77a99a2..f47a2e3 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -3,12 +3,12 @@ import json import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512", "url"], +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], 'format': 'misp_standard'} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '4', 'author': 'Hannah Ward', - 'description': 'Get information from virustotal', + 'description': 'Get information from VirusTotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin @@ -25,8 +25,7 @@ class VirusTotalParser(object): self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, 'domain': self.parse_domain, 'hostname': self.parse_domain, 'md5': self.parse_hash, 'sha1': self.parse_hash, - 'sha256': self.parse_hash, 'sha512': self.parse_hash, - 'url': self.parse_url} + 'sha256': self.parse_hash, 'url': self.parse_url} def query_api(self, attribute): self.attribute = MISPAttribute() diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index f31855e..e7c2e96 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -3,10 +3,10 @@ import json import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512", "url"], +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], 'format': 'misp_standard'} moduleinfo = {'version': '1', 'author': 'Christian Studer', - 'description': 'Get information from virustotal public API v2.', + 'description': 'Get information from VirusTotal public API v2.', 'module-type': ['expansion', 'hover']} moduleconfig = ['apikey'] @@ -155,7 +155,7 @@ ip = ('ip', IpQuery) file = ('resource', HashQuery) misp_type_mapping = {'domain': domain, 'hostname': domain, 'ip-src': ip, 'ip-dst': ip, 'md5': file, 'sha1': file, 'sha256': file, - 'sha512': file, 'url': ('resource', UrlQuery)} + 'url': ('resource', UrlQuery)} def parse_error(status_code): From 4e7192f7352ae339e88a18a77e0e359d9cb53e1b Mon Sep 17 00:00:00 2001 From: GlennHD Date: Wed, 12 Feb 2020 21:21:39 -0600 Subject: [PATCH 23/50] Added GeoIP City and GeoIP ASN Info --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index af78ca5..03adc27 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [GeoIP_City](misp_modules/modules/expansion/geoip_city.py) - a hover and expansion module to get GeoIP City information from geolite/maxmind. +* [GeoIP_ASN](misp_modules/modules/expansion/geoip_asn.py) - a hover and expansion module to get GeoIP ASN information from geolite/maxmind. * [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. * [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? From 7a3f9a422d9fd5767634be95a4e0443b8ea5bf9d Mon Sep 17 00:00:00 2001 From: GlennHD Date: Wed, 12 Feb 2020 21:28:41 -0600 Subject: [PATCH 24/50] Added GeoIP_City Enrichment module --- misp_modules/modules/expansion/geoip_city.py | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 misp_modules/modules/expansion/geoip_city.py diff --git a/misp_modules/modules/expansion/geoip_city.py b/misp_modules/modules/expansion/geoip_city.py new file mode 100644 index 0000000..9c9f847 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_city.py @@ -0,0 +1,64 @@ +import json +import geoip2.database +import sys +import logging + +log = logging.getLogger('geoip_city') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} +moduleconfig = ['local_geolite_db'] +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'GlennHD', + 'description': 'Query a local copy of the Maxmind Geolite City database (MMDB format)', + 'module-type': ['expansion', 'hover']} + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of Maxminds Geolite database'} + path_to_geolite = request['config']['local_geolite_db'] + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return False + + try: + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.city(toquery) + stringmap = 'Continent=' + str(answer.continent.name) + ', Country=' + str(answer.country.name) + ', Subdivision=' + str(answer.subdivisions.most_specific.name) + ', City=' + str(answer.city.name) + + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" + return misperrors + + r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 0b9b6c4f4164b306cbe2d986a2829857e4b307d3 Mon Sep 17 00:00:00 2001 From: GlennHD Date: Wed, 12 Feb 2020 21:29:40 -0600 Subject: [PATCH 25/50] Added GeoIP_ASN Enrichment module --- misp_modules/modules/expansion/geoip_asn.py | 63 +++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 misp_modules/modules/expansion/geoip_asn.py diff --git a/misp_modules/modules/expansion/geoip_asn.py b/misp_modules/modules/expansion/geoip_asn.py new file mode 100644 index 0000000..b7fa973 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_asn.py @@ -0,0 +1,63 @@ +import json +import geoip2.database +import sys +import logging + +log = logging.getLogger('geoip_asn') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} +moduleconfig = ['local_geolite_db'] +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'GlennHD', + 'description': 'Query a local copy of the Maxmind Geolite ASN database (MMDB format)', + 'module-type': ['expansion', 'hover']} + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of the Maxmind Geolite ASN database'} + path_to_geolite = request['config']['local_geolite_db'] + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return False + + try: + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.asn(toquery) + stringmap = 'ASN=' + str(answer.autonomous_system_number) + ', AS Org=' + str(answer.autonomous_system_organization) + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" + return misperrors + + r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 46f0f410e79be952bce3e58f289fdda5c83f7e80 Mon Sep 17 00:00:00 2001 From: GlennHD Date: Wed, 12 Feb 2020 21:31:41 -0600 Subject: [PATCH 26/50] Added geoip_asn and geoip_city to load --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 12c2ab6..458611f 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -6,7 +6,7 @@ sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/') __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', - 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', + 'whois', 'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', From bdb4185a0ada4c815ea11b29d1e4a5d08e381694 Mon Sep 17 00:00:00 2001 From: GlennHD Date: Wed, 12 Feb 2020 23:48:20 -0600 Subject: [PATCH 27/50] Update geoip_city.py --- misp_modules/modules/expansion/geoip_city.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/geoip_city.py b/misp_modules/modules/expansion/geoip_city.py index 9c9f847..01c0627 100644 --- a/misp_modules/modules/expansion/geoip_city.py +++ b/misp_modules/modules/expansion/geoip_city.py @@ -19,6 +19,7 @@ moduleinfo = {'version': '0.1', 'author': 'GlennHD', 'description': 'Query a local copy of the Maxmind Geolite City database (MMDB format)', 'module-type': ['expansion', 'hover']} + def handler(q=False): if q is False: return False From 0ed0ceab9d55835c78b51ee68b2c41f6f4e5dde3 Mon Sep 17 00:00:00 2001 From: GlennHD Date: Wed, 12 Feb 2020 23:48:38 -0600 Subject: [PATCH 28/50] Update geoip_asn.py --- misp_modules/modules/expansion/geoip_asn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/geoip_asn.py b/misp_modules/modules/expansion/geoip_asn.py index b7fa973..95d7ee7 100644 --- a/misp_modules/modules/expansion/geoip_asn.py +++ b/misp_modules/modules/expansion/geoip_asn.py @@ -19,6 +19,7 @@ moduleinfo = {'version': '0.1', 'author': 'GlennHD', 'description': 'Query a local copy of the Maxmind Geolite ASN database (MMDB format)', 'module-type': ['expansion', 'hover']} + def handler(q=False): if q is False: return False From 27717c040029e2d849f0e2c62a0240bdb18ae1d5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 13 Feb 2020 11:40:22 +0100 Subject: [PATCH 29/50] fix: Making the module config available so the module works --- misp_modules/modules/expansion/geoip_country.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 8ba012b..d28e570 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -59,5 +59,5 @@ def introspection(): def version(): - # moduleinfo['config'] = moduleconfig + moduleinfo['config'] = moduleconfig return moduleinfo From df3a6986ea5f2ba4ac73656037115e62c9c8c6d5 Mon Sep 17 00:00:00 2001 From: Mathilde Oun et Vincent Gindt Date: Fri, 21 Feb 2020 12:05:41 +0100 Subject: [PATCH 30/50] =?UTF-8?q?Rendu=20projet=20master2=20s=C3=A9curit?= =?UTF-8?q?=C3=A9=20par=20Mathilde=20OUN=20et=20Vincent=20GINDT=20//=20Nou?= =?UTF-8?q?veau=20module=20misp=20de=20recherche=20google=20sur=20les=20ur?= =?UTF-8?q?ls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/expansion/google_search.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 misp_modules/modules/expansion/google_search.py diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py new file mode 100644 index 0000000..2e8f2a8 --- /dev/null +++ b/misp_modules/modules/expansion/google_search.py @@ -0,0 +1,37 @@ +import json +import requests +try: + from google import google +except ImportError: + print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['url'], 'output': ['text']} +moduleinfo = {'author': 'Oun & Gindt', 'description': 'An expansion hover module to expand google search information about an URL', 'module-type': ['hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('url'): + toquery = request['url'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + num_page = 1 + res = "" + search_results = google.search(request['url'], num_page) + for i in range(3): + res += "("+str(i+1)+")" + '\t' + res += json.dumps(search_results[i].description, ensure_ascii=False) + res += '\n\n' + return {'results': [{'types': mispattributes['output'], 'values':res}]} + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo From f5af7faace15b9232948ef23f23408eab10123ab Mon Sep 17 00:00:00 2001 From: Sean Whalen <44679+seanthegeek@users.noreply.github.com> Date: Sat, 22 Feb 2020 19:44:31 -0500 Subject: [PATCH 31/50] Create __init__.py --- misp_modules/modules/expansion/_ransomcoindb/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 misp_modules/modules/expansion/_ransomcoindb/__init__.py diff --git a/misp_modules/modules/expansion/_ransomcoindb/__init__.py b/misp_modules/modules/expansion/_ransomcoindb/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/misp_modules/modules/expansion/_ransomcoindb/__init__.py @@ -0,0 +1 @@ + From 42dffa7291d84de2ea291ddec06b819aa5a5cee9 Mon Sep 17 00:00:00 2001 From: Sean Whalen <44679+seanthegeek@users.noreply.github.com> Date: Sun, 23 Feb 2020 15:24:18 -0500 Subject: [PATCH 32/50] Install cmake to build faup --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8ac6d9f..3d5efa7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,6 +21,7 @@ RUN set -eu \ libzbar0 \ libzbar-dev \ libfuzzy-dev \ + cmake \ ;apt-get -y autoremove \ ;apt-get -y clean \ ;rm -rf /var/lib/apt/lists/* \ From 180985f89cf8e9effcfb28399e76beafff3814f9 Mon Sep 17 00:00:00 2001 From: Sean Whalen <44679+seanthegeek@users.noreply.github.com> Date: Sun, 23 Feb 2020 15:34:02 -0500 Subject: [PATCH 33/50] Revert change inteded for other patch --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3d5efa7..8ac6d9f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,6 @@ RUN set -eu \ libzbar0 \ libzbar-dev \ libfuzzy-dev \ - cmake \ ;apt-get -y autoremove \ ;apt-get -y clean \ ;rm -rf /var/lib/apt/lists/* \ From dea42d39297b237d7b16ccbbcd2182af5d1a7561 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 25 Feb 2020 15:22:06 +0100 Subject: [PATCH 34/50] chg: Catching missing config issue --- misp_modules/modules/expansion/ransomcoindb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/ransomcoindb.py b/misp_modules/modules/expansion/ransomcoindb.py index 3bac983..2b9b566 100644 --- a/misp_modules/modules/expansion/ransomcoindb.py +++ b/misp_modules/modules/expansion/ransomcoindb.py @@ -26,6 +26,8 @@ def handler(q=False): return False q = json.loads(q) + if "config" not in q or "api-key" not in q["config"]: + return {"error": "Ransomcoindb API key is missing"} api_key = q["config"]["api-key"] r = {"results": []} From f9f3db84680dfa167ef9c4f35261d97499d5b052 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 25 Feb 2020 15:26:52 +0100 Subject: [PATCH 35/50] chg: Quick ransomdncoin test just to make sure the module loads - I do not have any api key right now, so the test should just reach the error --- tests/test_expansions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index ee3a906..801769a 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -363,6 +363,15 @@ class TestExpansions(unittest.TestCase): response = self.misp_modules_post(query) self.assertEqual(self.get_values(response), '1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh') + def test_ransomcoindb(self): + query = {"module": "ransomcoindb", + "attributes": {"type": "btc", + "value": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + if 'ransomcoindb' not in self.configs: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "Ransomcoindb API key is missing") + def test_rbl(self): query = {"module": "rbl", "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) From c9c6f69bd432e2a98129d02c3ad2b125fb321a80 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 26 Feb 2020 11:59:14 +0100 Subject: [PATCH 36/50] fix: Making pep8 happy --- .../expansion/_ransomcoindb/__init__.py | 1 - .../modules/expansion/google_search.py | 40 +++++++++---------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/misp_modules/modules/expansion/_ransomcoindb/__init__.py b/misp_modules/modules/expansion/_ransomcoindb/__init__.py index 8b13789..e69de29 100644 --- a/misp_modules/modules/expansion/_ransomcoindb/__init__.py +++ b/misp_modules/modules/expansion/_ransomcoindb/__init__.py @@ -1 +0,0 @@ - diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py index 2e8f2a8..067edaf 100644 --- a/misp_modules/modules/expansion/google_search.py +++ b/misp_modules/modules/expansion/google_search.py @@ -1,37 +1,35 @@ import json import requests try: - from google import google + from google import google except ImportError: - print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") + print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") misperrors = {'error': 'Error'} mispattributes = {'input': ['url'], 'output': ['text']} -moduleinfo = {'author': 'Oun & Gindt', 'description': 'An expansion hover module to expand google search information about an URL', 'module-type': ['hover']} +moduleinfo = {'author': 'Oun & Gindt', 'module-type': ['hover'], + 'description': 'An expansion hover module to expand google search information about an URL'} def handler(q=False): - if q is False: - return False - request = json.loads(q) - if request.get('url'): - toquery = request['url'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors + if q is False: + return False + request = json.loads(q) + if not request.get('url'): + return {'error': "Unsupported attributes type"} + num_page = 1 + res = "" + search_results = google.search(request['url'], num_page) + for i in range(3): + res += "("+str(i+1)+")" + '\t' + res += json.dumps(search_results[i].description, ensure_ascii=False) + res += '\n\n' + return {'results': [{'types': mispattributes['output'], 'values':res}]} - num_page = 1 - res = "" - search_results = google.search(request['url'], num_page) - for i in range(3): - res += "("+str(i+1)+")" + '\t' - res += json.dumps(search_results[i].description, ensure_ascii=False) - res += '\n\n' - return {'results': [{'types': mispattributes['output'], 'values':res}]} def introspection(): - return mispattributes + return mispattributes def version(): - return moduleinfo + return moduleinfo From cda5004a0ddf1aa167fd5ad01f5b2f4ea2bcb6e5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 26 Feb 2020 14:18:09 +0100 Subject: [PATCH 37/50] fix: Removed unused import --- misp_modules/modules/expansion/google_search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py index 067edaf..b7b4e7a 100644 --- a/misp_modules/modules/expansion/google_search.py +++ b/misp_modules/modules/expansion/google_search.py @@ -1,5 +1,4 @@ import json -import requests try: from google import google except ImportError: From a32685df8af9c9ae50db9bf214f8759a44084b5c Mon Sep 17 00:00:00 2001 From: bennyv Date: Wed, 4 Mar 2020 09:52:55 +1100 Subject: [PATCH 38/50] Initial Build of SOPHOSLabs Intelix Product --- misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/sophoslabs_intelix.py | 125 ++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/sophoslabs_intelix.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 458611f..8eb0a92 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix'] diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py new file mode 100644 index 0000000..4cc65ca --- /dev/null +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -0,0 +1,125 @@ +from pymisp import MISPAttribute, MISPEvent, MISPObject +import json +import requests +import base64 +from urllib.parse import quote + +moduleinfo = {'version': '1.0', + 'author': 'Ben Verschaeren', + 'description': 'SOPHOSLabs Intelix Integration', + 'module-type': ['expansion']} + +moduleconfig = ['client_id', 'client_secret'] + +misperrors = {'error': 'Error'} + +misp_types_in = ['sha256', 'ip', 'ip-src', 'ip-dst', 'uri', 'url', 'domain', 'hostname'] + +mispattributes = {'input': misp_types_in, + 'format': 'misp_standard'} + +class SophosLabsApi(): + def __init__(self, client_id, client_secret): + self.misp_event = MISPEvent() + self.client_id = client_id + self.client_secret= client_secret + self.authToken = f"{self.client_id}:{self.client_secret}" + self.baseurl = 'de.api.labs.sophos.com' + d = {'grant_type': 'client_credentials'} + h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}",\ + 'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.post('https://api.labs.sophos.com/oauth2/token', headers=h, data=d) + if r.status_code == 200: + j = json.loads(r.text) + self.accessToken = j['access_token'] + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def hash_lookup(self, filehash): + sophos_object = MISPObject('SOPHOSLabs Intelix SHA256 Report') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/files/v1/{filehash}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'reputationScore' in j: + sophos_object.add_attribute('Reputation Score', type='text', value=j['reputationScore']) + if 0 <= j['reputationScore'] <= 19: + sophos_object.add_attribute('Decision', type='text', value='This file is malicious') + if 20 <= j['reputationScore'] <= 29: + sophos_object.add_attribute('Decision', type='text', value='This file is potentially unwanted') + if 30 <= j['reputationScore'] <= 69: + sophos_object.add_attribute('Decision', type='text', value='This file is unknown and suspicious') + if 70 <= j['reputationScore'] <= 100: + sophos_object.add_attribute('Decision', type='text', value='This file is known good') + if 'detectionName' in j: + sophos_object.add_attribute('Detection Name', type='text', value=j['detectionName']) + else: + sophos_object.add_attribute('Detection Name', type='text', value='No name associated with this IoC') + self.misp_event.add_object(**sophos_object) + + def ip_lookup(self, ip): + sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'category' in j: + for c in j['category']: + sophos_object.add_attribute('IP Address Categorisation', type='text', value=c) + else: + sophos_object.add_attribute('IP Address Categorisation', type='text', value='No category assocaited with IoC') + self.misp_event.add_object(**sophos_object) + + def url_lookup(self, url): + sophos_object = MISPObject('SOPHOSLabs Intelix URL Lookup') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/urls/v1/{quote(url, safe='')}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'productivityCategory' in j: + sophos_object.add_attribute('URL Categorisation', type='text', value=j['productivityCategory']) + else: + sophos_object.add_attribute('URL Categorisation', type='text', value='No category assocaited with IoC') + + if 'riskLevel' in j: + sophos_object.add_attribute('URL Risk Level', type='text', value=j['riskLevel']) + else: + sophos_object.add_attribute('URL Risk Level', type='text', value='No risk level associated with IoC') + + if 'securityCategory' in j: + sophos_object.add_attribute('URL Security Category', type='text', value=j['securityCategory']) + else: + sophos_object.add_attribute('URL Security Category', type='text', value='No Security Category associated with IoC') + self.misp_event.add_object(**sophos_object) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request['config']['client_id'] is None or request['config']['client_secret'] is None: + misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \ + It's free to Sign Up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." + return misperrors + else: + client = SophosLabsApi(request['config']['client_id'], request['config']['client_secret']) + if request['attribute']['type'] == "sha256": + client.hash_lookup(request['attribute']['value1']) + if request['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']: + client.ip_lookup(request["attribute"]["value1"]) + if request['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']: + client.url_lookup(request["attribute"]["value1"]) + return client.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + From 277f56e088c5c63e16d86da7f74aa0638e5821de Mon Sep 17 00:00:00 2001 From: bennyv Date: Wed, 4 Mar 2020 10:39:35 +1100 Subject: [PATCH 39/50] Updated the README.md for SOPHOSLabs Intelix --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 03adc27..f4c3156 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. +* [SOPHOSLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SOPHOSLabs Intelix is an API for Threat Intelligence and Analysis (Free tier)[SOPHOSLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). From 4771a5177de39ee0f8a04f72091729b35663c73c Mon Sep 17 00:00:00 2001 From: bennyv Date: Wed, 4 Mar 2020 10:43:24 +1100 Subject: [PATCH 40/50] Fixed formatting in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4c3156..996f2d0 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. -* [SOPHOSLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SOPHOSLabs Intelix is an API for Threat Intelligence and Analysis (Free tier)[SOPHOSLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) +* [SophosLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SophosLabs Intelix is an API for Threat Intelligence and Analysis (free tier availible). [SophosLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). From 0a8a829ac10650b5eb41e0e4d0e9f8b6606909b6 Mon Sep 17 00:00:00 2001 From: bennyv Date: Wed, 4 Mar 2020 11:30:44 +1100 Subject: [PATCH 41/50] Fixed handler error handling for missing config --- .../modules/expansion/sophoslabs_intelix.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py index 4cc65ca..e4dcab6 100644 --- a/misp_modules/modules/expansion/sophoslabs_intelix.py +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -99,20 +99,19 @@ class SophosLabsApi(): def handler(q=False): if q is False: return False - request = json.loads(q) - if request['config']['client_id'] is None or request['config']['client_secret'] is None: + j = json.loads(q) + if not j.get('config') or not j['config'].get('client_id') or not j['config'].get('client_secret'): misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \ - It's free to Sign Up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." + It's free to sign up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." return misperrors - else: - client = SophosLabsApi(request['config']['client_id'], request['config']['client_secret']) - if request['attribute']['type'] == "sha256": - client.hash_lookup(request['attribute']['value1']) - if request['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']: - client.ip_lookup(request["attribute"]["value1"]) - if request['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']: - client.url_lookup(request["attribute"]["value1"]) - return client.get_result() + client = SophosLabsApi(j['config']['client_id'], j['config']['client_secret']) + if j['attribute']['type'] == "sha256": + client.hash_lookup(j['attribute']['value1']) + if j['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']: + client.ip_lookup(j["attribute"]["value1"]) + if j['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']: + client.url_lookup(j["attribute"]["value1"]) + return client.get_result() def introspection(): From 6c00f02e42959c635d3b5a4b3bed5da67a845ce2 Mon Sep 17 00:00:00 2001 From: bennyv Date: Wed, 4 Mar 2020 11:54:55 +1100 Subject: [PATCH 42/50] Removed Unused Import --- misp_modules/modules/expansion/sophoslabs_intelix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py index e4dcab6..57a1af0 100644 --- a/misp_modules/modules/expansion/sophoslabs_intelix.py +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -1,4 +1,4 @@ -from pymisp import MISPAttribute, MISPEvent, MISPObject +from pymisp import MISPEvent, MISPObject import json import requests import base64 From 0b4d6738de501526d99fb7e9eebfc08dfd4722cf Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 10 Mar 2020 11:15:16 +0100 Subject: [PATCH 43/50] fix: Making pep8 happy --- misp_modules/modules/expansion/sophoslabs_intelix.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py index 57a1af0..017683a 100644 --- a/misp_modules/modules/expansion/sophoslabs_intelix.py +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -18,16 +18,17 @@ misp_types_in = ['sha256', 'ip', 'ip-src', 'ip-dst', 'uri', 'url', 'domain', 'ho mispattributes = {'input': misp_types_in, 'format': 'misp_standard'} + class SophosLabsApi(): def __init__(self, client_id, client_secret): self.misp_event = MISPEvent() self.client_id = client_id - self.client_secret= client_secret + self.client_secret = client_secret self.authToken = f"{self.client_id}:{self.client_secret}" self.baseurl = 'de.api.labs.sophos.com' d = {'grant_type': 'client_credentials'} - h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}",\ - 'Content-Type': 'application/x-www-form-urlencoded'} + h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}", + 'Content-Type': 'application/x-www-form-urlencoded'} r = requests.post('https://api.labs.sophos.com/oauth2/token', headers=h, data=d) if r.status_code == 200: j = json.loads(r.text) @@ -83,12 +84,12 @@ class SophosLabsApi(): sophos_object.add_attribute('URL Categorisation', type='text', value=j['productivityCategory']) else: sophos_object.add_attribute('URL Categorisation', type='text', value='No category assocaited with IoC') - + if 'riskLevel' in j: sophos_object.add_attribute('URL Risk Level', type='text', value=j['riskLevel']) else: sophos_object.add_attribute('URL Risk Level', type='text', value='No risk level associated with IoC') - + if 'securityCategory' in j: sophos_object.add_attribute('URL Security Category', type='text', value=j['securityCategory']) else: @@ -121,4 +122,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - From e023f0b4700122d18f94a7e5663b2a712babfcb5 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 10 Mar 2020 18:25:30 +0100 Subject: [PATCH 44/50] Cytomic Orion MISP Module An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/cytomic_orion.py | 183 ++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/expansion/cytomic_orion.py diff --git a/README.md b/README.md index 996f2d0..fe37cd5 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). * [CVE advanced](misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). * [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. +* [Cytomic Orion](misp_modules/modules/expansion/cytomic_orion.py) - An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [docx-enrich](misp_modules/modules/expansion/docx_enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 8eb0a92..53c8ad8 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit', 'sophoslabs_intelix'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion.py'] diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py new file mode 100755 index 0000000..897840b --- /dev/null +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +''' +Cytomic Orion MISP Module +An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion + + +''' + +from pymisp import MISPAttribute, MISPEvent, MISPObject, MISPTag +import json +import requests +import re + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['md5'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.3', 'author': 'Koen Van Impe', + 'description': 'an expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion', + 'module-type': ['expansion']} +moduleconfig = ['api_url', 'token_url', 'clientid', 'clientsecret', 'clientsecret', 'username', 'password', 'upload_timeframe', 'upload_tag', 'delete_tag', 'upload_ttlDays', 'upload_threat_level_id', 'limit_upload_events', 'limit_upload_attributes'] +# There are more config settings in this module than used by the enrichment +# There is also a PyMISP module which reuses the module config, and requires additional configuration, for example used for pushing indicators to the API + + +class CytomicParser(): + def __init__(self, attribute, config_object): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + + self.config_object = config_object + + if self.config_object: + self.token = self.get_token() + else: + return {'error': 'Missing configuration'} + + def get_token(self): + try: + scope = self.config_object['scope'] + grant_type = self.config_object['grant_type'] + username = self.config_object['username'] + password = self.config_object['password'] + token_url = self.config_object['token_url'] + clientid = self.config_object['clientid'] + clientsecret = self.config_object['clientsecret'] + + if scope and grant_type and username and password: + data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} + + if token_url and clientid and clientsecret: + access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) + tokens = json.loads(access_token_response.text) + if 'access_token' in tokens: + return tokens['access_token'] + else: + self.result = {'error': 'No token received.'} + return + else: + self.result = {'error': 'No token_url, clientid or clientsecret supplied.'} + return + else: + self.result = {'error': 'No scope, grant_type, username or password supplied.'} + return + except Exception: + self.result = {'error': 'Unable to connect to token_url.'} + return + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse(self, searchkey): + + if self.token: + + endpoint_fileinformation = self.config_object['endpoint_fileinformation'] + endpoint_machines = self.config_object['endpoint_machines'] + endpoint_machines_client = self.config_object['endpoint_machines_client'] + query_machines = self.config_object['query_machines'] + query_machine_info = self.config_object['query_machine_info'] + + # Update endpoint URLs + query_endpoint_fileinformation = endpoint_fileinformation.format(md5=searchkey) + query_endpoint_machines = endpoint_machines.format(md5=searchkey) + + # API calls + api_call_headers = {'Authorization': 'Bearer ' + self.token} + result_query_endpoint_fileinformation = requests.get(query_endpoint_fileinformation, headers=api_call_headers, verify=False) + json_result_query_endpoint_fileinformation = json.loads(result_query_endpoint_fileinformation.text) + + if json_result_query_endpoint_fileinformation: + + cytomic_object = MISPObject('cytomic-orion-file') + + cytomic_object.add_attribute('fileName', type='text', value=json_result_query_endpoint_fileinformation['fileName']) + cytomic_object.add_attribute('fileSize', type='text', value=json_result_query_endpoint_fileinformation['fileSize']) + cytomic_object.add_attribute('last-seen', type='datetime', value=json_result_query_endpoint_fileinformation['lastSeen']) + cytomic_object.add_attribute('first-seen', type='datetime', value=json_result_query_endpoint_fileinformation['firstSeen']) + cytomic_object.add_attribute('classification', type='text', value=json_result_query_endpoint_fileinformation['classification']) + cytomic_object.add_attribute('classificationName', type='text', value=json_result_query_endpoint_fileinformation['classificationName']) + self.misp_event.add_object(**cytomic_object) + + result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) + json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) + + if json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + for machine in json_result_query_endpoint_machines: + + if machine['muid'] and query_machine_info: + query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) + result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) + json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) + + if json_result_endpoint_machines_client: + + cytomic_machine_object = MISPObject('cytomic-orion-machine') + + clienttag = [{'name': json_result_endpoint_machines_client['clientName']}] + + cytomic_machine_object.add_attribute('machineName', type='target-machine', value=json_result_endpoint_machines_client['machineName'], Tag=clienttag) + cytomic_machine_object.add_attribute('machineMuid', type='text', value=machine['muid']) + cytomic_machine_object.add_attribute('clientName', type='target-org', value=json_result_endpoint_machines_client['clientName'], Tag=clienttag) + cytomic_machine_object.add_attribute('clientId', type='text', value=machine['clientId']) + cytomic_machine_object.add_attribute('machinePath', type='text', value=machine['lastPath']) + cytomic_machine_object.add_attribute('first-seen', type='datetime', value=machine['firstSeen']) + cytomic_machine_object.add_attribute('last-seen', type='datetime', value=machine['lastSeen']) + cytomic_machine_object.add_attribute('creationDate', type='datetime', value=json_result_endpoint_machines_client['creationDate']) + cytomic_machine_object.add_attribute('clientCreationDateUTC', type='datetime', value=json_result_endpoint_machines_client['clientCreationDateUTC']) + cytomic_machine_object.add_attribute('lastSeenUtc', type='datetime', value=json_result_endpoint_machines_client['lastSeenUtc']) + self.misp_event.add_object(**cytomic_machine_object) + else: + self.result = {'error': 'No (valid) token.'} + return + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + + if not request.get('config'): + return {'error': 'Missing configuration'} + + config_object = { + 'clientid': request["config"].get("clientid"), + 'clientsecret': request["config"].get("clientsecret"), + 'scope': 'orion.api', + 'password': request["config"].get("password"), + 'username': request["config"].get("username"), + 'grant_type': 'password', + 'token_url': request["config"].get("token_url"), + 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), + 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), + 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), + 'query_machines': True, + 'query_machine_info': True + } + + cytomic_parser = CytomicParser(attribute, config_object) + cytomic_parser.parse(attribute['value']) + + return cytomic_parser.get_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From c86f4a418053718eea07e1baa64d7d5db096a3d4 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 10 Mar 2020 18:48:25 +0100 Subject: [PATCH 45/50] Make Travis (a little bit) happy --- misp_modules/modules/expansion/cytomic_orion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py index 897840b..c7830e8 100755 --- a/misp_modules/modules/expansion/cytomic_orion.py +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -7,10 +7,10 @@ An expansion module to enrich attributes in MISP and share indicators of comprom ''' -from pymisp import MISPAttribute, MISPEvent, MISPObject, MISPTag +from pymisp import MISPAttribute, MISPEvent, MISPObject import json import requests -import re +import sys misperrors = {'error': 'Error'} mispattributes = {'input': ['md5'], 'format': 'misp_standard'} @@ -34,7 +34,7 @@ class CytomicParser(): if self.config_object: self.token = self.get_token() else: - return {'error': 'Missing configuration'} + sys.exit('Missing configuration') def get_token(self): try: @@ -108,10 +108,10 @@ class CytomicParser(): result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) - if json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + if query_machines and json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: for machine in json_result_query_endpoint_machines: - if machine['muid'] and query_machine_info: + if query_machine_info and machine['muid']: query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) From 2713d3c6555f3e7f36a8aa0364a74a71ec1bd85a Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 10 Mar 2020 19:50:00 +0100 Subject: [PATCH 46/50] Update __init__ --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 53c8ad8..2a99050 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion.py'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion'] From d2f0d8027bd3f380198a22bbfb7ca300fd39b1fb Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 11:56:12 +0100 Subject: [PATCH 47/50] Documentation for Cytomic Orion --- doc/expansion/cytomic_orion.py | 9 +++++++++ doc/logos/cytomic_orion.png | Bin 0 -> 898 bytes 2 files changed, 9 insertions(+) create mode 100644 doc/expansion/cytomic_orion.py create mode 100644 doc/logos/cytomic_orion.png diff --git a/doc/expansion/cytomic_orion.py b/doc/expansion/cytomic_orion.py new file mode 100644 index 0000000..6f87657 --- /dev/null +++ b/doc/expansion/cytomic_orion.py @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to enrich attributes in MISP by quering the Cytomic Orion API", + "logo": "logos/cytomic_orion.png", + "requirements": ["Access (license) to Cytomic Orion"], + "input": "MD5, hash of the sample / malware to search for.", + "output": "MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.", + "references": ["https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", "https://www.cytomicmodel.com/solutions/"], + "features": "This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines." +} diff --git a/doc/logos/cytomic_orion.png b/doc/logos/cytomic_orion.png new file mode 100644 index 0000000000000000000000000000000000000000..45704e9278088bb5305b3b0b20fcb4158b8fe2ee GIT binary patch literal 898 zcmV-|1AY97P)fVtcD? zYC!oEge*lRQ+<$8WgRLQmX~&DAQ85>=1)@bO%R&r)SEd4S*k8mCidD-B(Y&t=*w~O z;XcTeR4$)D$+J2)=BFSvN6ND|M32|YY?V2PiyQjV5Nq}ZH~WGw9F^OSNMlH3F?>0Q z@M}P>toG{WoDSjU4Yc_bWRt1fD#q4Gya7Tas1l?LVoPKTTb+PjmogaJD4YdTYTlZb zTOf0LGTGWRr^DO;Td?kexJ+sgQ75me@*y$5@)V?-^gAGHOjLcSJqB?jZSHq& z5la3U2oGU;3Tv==+&u(wsWhjeo5JiIWIy!~8udwIkjq+iE&2|K@VjBS|D1z#7yI@0 zYjnUR`DKuB6AdGC4#L_|y;Lfc_s_WDs(-pHR6l<%iXj?D}4NAl(nE7D=@~eG1aa z4GaqJ=UtHQr*8GJt9r@%35d(3xB*iBD53dfSZL718|U7yLCk6B&+H43fx!tRE3aao z464#g`}l7Be7mDA2n;&MpK{{z2c6ip^SU>^{zP366bgkxp-?Ck3WY+UP$(1%h5AeS Y2jOxQJ{Y2W1^@s607*qoM6N<$g6^}Z-~a#s literal 0 HcmV?d00001 From fe34023866da011b175709a862d6efec579b56b3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 12 Mar 2020 11:02:43 +0100 Subject: [PATCH 48/50] csvimport: Return error if input is not valid UTF-8 --- misp_modules/modules/import_mod/csvimport.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 8bfbbe9..38e5f96 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -256,7 +256,11 @@ def handler(q=False): return False request = json.loads(q) if request.get('data'): - data = base64.b64decode(request['data']).decode('utf-8') + try: + data = base64.b64decode(request['data']).decode('utf-8') + except UnicodeDecodeError: + misperrors['error'] = "Input is not valid UTF-8" + return misperrors else: misperrors['error'] = "Unsupported attributes type" return misperrors From 422f654988c94a32d78a4e0aa81d7785497ea718 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 18 Mar 2020 10:24:06 +0100 Subject: [PATCH 49/50] fix: Making pep8 happy with indentation --- .../modules/expansion/cytomic_orion.py | 26 +++++++++---------- misp_modules/modules/import_mod/csvimport.py | 6 ++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py index c7830e8..9723ed6 100755 --- a/misp_modules/modules/expansion/cytomic_orion.py +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -154,19 +154,19 @@ def handler(q=False): return {'error': 'Missing configuration'} config_object = { - 'clientid': request["config"].get("clientid"), - 'clientsecret': request["config"].get("clientsecret"), - 'scope': 'orion.api', - 'password': request["config"].get("password"), - 'username': request["config"].get("username"), - 'grant_type': 'password', - 'token_url': request["config"].get("token_url"), - 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), - 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), - 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), - 'query_machines': True, - 'query_machine_info': True - } + 'clientid': request["config"].get("clientid"), + 'clientsecret': request["config"].get("clientsecret"), + 'scope': 'orion.api', + 'password': request["config"].get("password"), + 'username': request["config"].get("username"), + 'grant_type': 'password', + 'token_url': request["config"].get("token_url"), + 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), + 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), + 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), + 'query_machines': True, + 'query_machine_info': True + } cytomic_parser = CytomicParser(attribute, config_object) cytomic_parser.parse(attribute['value']) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 38e5f96..e4dc2e5 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -257,10 +257,10 @@ def handler(q=False): request = json.loads(q) if request.get('data'): try: - data = base64.b64decode(request['data']).decode('utf-8') + data = base64.b64decode(request['data']).decode('utf-8') except UnicodeDecodeError: - misperrors['error'] = "Input is not valid UTF-8" - return misperrors + misperrors['error'] = "Input is not valid UTF-8" + return misperrors else: misperrors['error'] = "Unsupported attributes type" return misperrors From 824c0031b3aa867284a00ecf60ad992e86d6b5ef Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 18 Mar 2020 17:57:55 +0100 Subject: [PATCH 50/50] fix: Catching errors in the reponse of the query to URLhaus --- misp_modules/modules/expansion/urlhaus.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py index 30b78ee..baaaaf6 100644 --- a/misp_modules/modules/expansion/urlhaus.py +++ b/misp_modules/modules/expansion/urlhaus.py @@ -35,6 +35,11 @@ class URLhaus(): results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} + def parse_error(self, query_status): + if query_status == 'no_results': + return {'error': f'No results found on URLhaus for this {self.attribute.type} attribute'} + return {'error': f'Error encountered during the query of URLhaus: {query_status}'} + class HostQuery(URLhaus): def __init__(self, attribute): @@ -45,9 +50,12 @@ class HostQuery(URLhaus): def query_api(self): response = requests.post(self.url, data={'host': self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) if 'urls' in response and response['urls']: for url in response['urls']: self.misp_event.add_attribute(type='url', value=url['url']) + return self.get_result() class PayloadQuery(URLhaus): @@ -63,6 +71,8 @@ class PayloadQuery(URLhaus): if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0': file_object.id = self.attribute.object_id response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')): if response[key]: @@ -81,6 +91,7 @@ class PayloadQuery(URLhaus): file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]}) if any((file_object.attributes, file_object.references)): self.misp_event.add_object(**file_object) + return self.get_result() class UrlQuery(URLhaus): @@ -100,6 +111,8 @@ class UrlQuery(URLhaus): def query_api(self): response = requests.post(self.url, data={'url': self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) if 'payloads' in response and response['payloads']: for payload in response['payloads']: file_object = self._create_file_object(payload) @@ -109,6 +122,7 @@ class UrlQuery(URLhaus): self.misp_event.add_object(**vt_object) if any((file_object.attributes, file_object.references)): self.misp_event.add_object(**file_object) + return self.get_result() _misp_type_mapping = {'url': UrlQuery, 'md5': PayloadQuery, 'sha256': PayloadQuery, @@ -122,8 +136,7 @@ def handler(q=False): request = json.loads(q) attribute = request['attribute'] urlhaus_parser = _misp_type_mapping[attribute['type']](attribute) - urlhaus_parser.query_api() - return urlhaus_parser.get_result() + return urlhaus_parser.query_api() def introspection():