From 69deb8d10ba73270db53735618f20de100c97590 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 21 Feb 2023 13:04:24 +0100 Subject: [PATCH 1/4] add: [ipinfo] First version of a new module to query ipinfo.io - First version addressing the request from #600 - Straight forward parsing of the `geolocation`, `domain-ip` and `asn` information returned by the standard API endpoint (ipinfo.io/{ip_address}) --- misp_modules/modules/expansion/ipinfo.py | 105 +++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 misp_modules/modules/expansion/ipinfo.py diff --git a/misp_modules/modules/expansion/ipinfo.py b/misp_modules/modules/expansion/ipinfo.py new file mode 100644 index 0000000..e83f4ad --- /dev/null +++ b/misp_modules/modules/expansion/ipinfo.py @@ -0,0 +1,105 @@ +import json +import requests +from . import check_input_attribute, standard_error_message +from pymisp import MISPAttribute, MISPEvent, MISPObject + +mispattributes = { + 'input': ['ip-src', 'ip-dst'], + 'format': 'misp_standard' +} +moduleinfo = { + 'version': 1, + 'author': 'Christian Studer', + 'description': 'An expansion module to query ipinfo.io for additional information on an IP address', + 'module-type': ['expansion', 'hover'] +} +moduleconfig = ['token'] + +_GEOLOCATION_OBJECT_MAPPING = { + 'city': 'city', + 'postal': 'zipcode', + 'region': 'region', + 'country': 'countrycode' +} + + +def handler(q=False): + # Input checks + if q is False: + return False + request = json.loads(q) + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] + if attribute.get('type') not in mispattributes['input']: + return {'error': 'Wrong input attribute type.'} + if not request.get('config'): + return {'error': 'Missing ipinfo config.'} + if not request['config'].get('token'): + return {'error': 'Missing ipinfo token.'} + + # Query ipinfo.io + query = requests.get( + f"https://ipinfo.io/{attribute['value']}/json?token={request['config']['token']}" + ) + if query.status_code != 200: + return {'error': f'Error while querying ipinfo.io - {query.status_code}: {query.reason}'} + ipinfo = query.json() + + # Check if the IP address is not reserved for special use + if ipinfo.get('bogon', False): + return {'error': 'The IP address is reserved for special use'} + + # Initiate the MISP data structures + misp_event = MISPEvent() + input_attribute = MISPAttribute() + input_attribute.from_dict(**attribute) + misp_event.add_attribute(**input_attribute) + + # Parse the geolocation information related to the IP address + geolocation = MISPObject('geolocation') + for field, relation in _GEOLOCATION_OBJECT_MAPPING.items(): + geolocation.add_attribute(relation, ipinfo[field]) + for relation, value in zip(('latitude', 'longitude'), ipinfo['loc'].split(',')): + geolocation.add_attribute(relation, value) + geolocation.add_reference(input_attribute.uuid, 'locates') + misp_event.add_object(geolocation) + + # Parse the domain information + domain_ip = misp_event.add_object(name='domain-ip') + for feature in ('hostname', 'ip'): + domain_ip.add_attribute(feature, ipinfo[feature]) + domain_ip.add_reference(input_attribute.uuid, 'resolves') + if ipinfo.get('domain') is not None: + for domain in ipinfo['domain']['domains']: + domain_ip.add_attribute('domain', domain) + + # Parse the AS information + asn = MISPObject('asn') + asn.add_reference(input_attribute.uuid, 'includes') + if ipinfo.get('asn') is not None: + asn_info = ipinfo['asn'] + asn.add_attribute('asn', asn_info['asn']) + asn.add_attribute('description', asn_info['name']) + misp_event.add_object(asn) + elif ipinfo.get('org'): + as_value, *description = ipinfo['org'].split(' ') + asn.add_attribute('asn', as_value) + asn.add_attribute('description', ' '.join(description)) + misp_event.add_object(asn) + + + # Return the results in MISP format + event = json.loads(misp_event.to_json()) + return { + 'results': {key: event[key] for key in ('Attribute', 'Object')} + } + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From f40f61fc189398c1e33a026673a08604dca78cff Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 21 Feb 2023 15:21:56 +0100 Subject: [PATCH 2/4] add: [documentation] Added documentation for the new ipinfo.io module & updated the main readme file --- README.md | 1 + documentation/README.md | 25 ++++++++++++++++++++ documentation/logos/ipinfo.png | Bin 0 -> 4948 bytes documentation/mkdocs/expansion.md | 25 ++++++++++++++++++++ documentation/website/expansion/ipinfo.json | 13 ++++++++++ 5 files changed, 64 insertions(+) create mode 100644 documentation/logos/ipinfo.png create mode 100644 documentation/website/expansion/ipinfo.json diff --git a/README.md b/README.md index 1f6b350..2deeefd 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [HYAS Insight](misp_modules/modules/expansion/hyasinsight.py) - a hover and expansion module to get information from [HYAS Insight](https://www.hyas.com/hyas-insight). * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. +* [ipinfo.io](misp_modules/modules/expansion/ipinfo.py) - an expansion module to get additional information on an IP address using the ipinfo.io API * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [Joe Sandbox submit](misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox. * [Joe Sandbox query](misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data. diff --git a/documentation/README.md b/documentation/README.md index 524e1a2..859034c 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -776,6 +776,31 @@ Module to query an IP ASN history service (https://github.com/D4-project/IPASN-H ----- +#### [ipinfo](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipinfo.py) + + + +An expansion module to query ipinfo.io to gather more information on a given IP address. +- **features**: +>The module takes an IP address attribute as input and queries the ipinfo.io API. +>The geolocation information on the IP address is always returned. +> +>Depending on the subscription plan, the API returns different pieces of information then: +>- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field. +>- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse. +> +>More information on the responses content is available in the [documentation](https://ipinfo.io/developers). +- **input**: +>IP address attribute. +- **output**: +>Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s). +- **references**: +>https://ipinfo.io/developers +- **requirements**: +>An ipinfo.io token + +----- + #### [ipqs_fraud_and_risk_scoring](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py) diff --git a/documentation/logos/ipinfo.png b/documentation/logos/ipinfo.png new file mode 100644 index 0000000000000000000000000000000000000000..9dedd955759bbd2485c82adb0ef7dbf89f527419 GIT binary patch literal 4948 zcmV-a6RYfrP)SIj*hFVtLy9Qh=_=Me0-y$qibtx&d$zTTU#nBDwdX( z*VorIH8ny)LTznr@b~&+Vq&zkwAkkG!NI{zO--%I-7zsSW}Ct2@%F*j=8v_}bfw3< z)Z{OGubjQs7ILafg|nf-*jbak(cS5gw$gs9%O`uUx^0Z?000tCNkliNJpyE=yJraoQ0tjfuj<_r(oE*0m+9`A935?q{%|tA z4=|k!``cfZ;Gme?e=G_$8)E+YGD!&fS&Z)=(vIy|X$-4i+s^0n@TZ6OXVJf36zoD; zfh9f8=ll4l2l*ESWJF;uzv|lVj>g<8aC%O z(VTBW|Ird0zZM1SDe*bqfyvDN3NLH3Hh=l)VgCVq*k96^kR1CpQJlkKy!f)pY~akN zKRx*WH70Y_I>uUt%2r{C4KXFC1xru_c!c`lOUBzm)d+*4*OC;g}oi2L_*A)Axfw7p|WgXV(sxJ60k z(V`(;5)14ZLlP2OwZ(RcvGD`-P__F5kM(b%Kc~!WpLm(33*_1)+qp+PV4qQ#e`R+2 zuh>ml^Q1VvH;@p(S$uJBpYZke!(oQ`2QkKuCdT6A8FN!h(rJ6bD{>aG!e6{7vOW`& zwiis0qbVqLravBzl~ewKxUoMDGt|F^1vF=JAKrg^U(DyrkLUB*(^MKGJfMY<6&*(u z>+MqM@_5I=moH1D_FyLw#d0&(ZC9@b*E^1dB7c$V_NuM;0`B3QTDD|PgXS71NM{EB zS&p4v+>2>~;tZNaCvxH0>KLZ$`o>K${?L!IBf}4bjPkKX6tDgnp6QytVffu@c89@G zlvBYSdPXI#0KBA;tY;>R0&Nwr?77$tFhpO9Qk$%2rdBX6HS{uO`qZw1eJ@hmMAp{sMY*}-S<;>p&NzmOysZG}m=VOQIs2WVjO-vDO zCSoP#BKBA)iWx?^5!2w#z{Fc=>S&&Y92r-z1yvjz>1k7=$9zWIZd_}WH8Xrt z#LPaasdLHZD3$~ip4r5l^&fPd0#2GV2aciPvjwe!fKy_`b-!gOB-WEz?7At4JyLV& zGpU(WtmozxP-D&TXZ=@w4~ONDOy)cIet`(XsQO?&;znr~keXO#F^iREpVW*@AU4j! zH->d&2QT8Imq7tPlFYag#j2L=rPgx@a7%8xMeLN8MKdCM9FE zVhntZvJr-8Mm?5*Mrc>vUhF}rz#@CsjiWTM?2PkX{ZC3mA!t8ygeE6k)O-VL8%G%!^z9S!7H7@ z3@IDPEm->E!d1#xqI4H$+zj=9fsjsIdJjrXvEeSq=AE=+T@BWoI0zt0W6!y63?E2w z2C2;l2*&3T9qPyB^6s|1RW7$;UtDF>#7=bSP&^*QzC&n!P-X??JkvFtDGhy_(h08bU46+f%I**$J7}x~d)-D^cyeC2W5BNFS+f#d0Y+jLvS?47 z8ZnQ}ac6|+u#u10-&UdU^=&XAZYGs+eZw%DVhEaL7x3(6ll`f>`butj={ibZ3b_Io z9~vyFEK)Hjt7;WsCI^7wj2En=F|>RMQAnqLw2s zYJxZoSu~)DQc^cysb{yusL6#h6;N|tH9(#%SX3Hk^cJ(3Ot?5=)Kp;5MTBIG4(GQ; z@@K2xzz=xw%rJ2N(>FTXuAo>F$UG6KDV~}{uxX@D;n!ZdSt=l_H0SfKc=U%rZBW7* zq_TAlX;IYN`Cf`(IkDuKWz?kP)C>$N#N-Jb2{kE9&4p`Zgs-?px23!*2u6)7P&4RA z1;g6|&K|7~p{ubKjY}XBK+XnUtBIcV)C}t6p$bQjAK7e5Lyg}&lTJX$O%yf78|fGV zdbf%h)YMcIf?cD^zWE8Bd>Bo^Ctp<(0}B|WSRj{erPWnvEj8|mDNbsD4CY2l&2pmU zA~yacf|~QL3Ak-ev@8Kn--|(=A;2vn_(lN7DFM@)ak$!yMs7PjUCu#~xYSig~t*CKVA+3k3n-2=@ z+fcI>0oBlNufEtaJn(1N55R)wD&oyGea|2CkSG^kCT{PM$=+8m9E5g?V(v91TR<_(gj>A|@Rkm8G^uk$jPi`)kxZR9p z0Ke`V-P>yAru`2F;u$|GIVPp%7z#FxPPA%WkG5)oNOFk$DET5zk&pIB~7q70e|*5RNa%u7Tp zhlYO*@aUtp)LjY;Bcco=qXrbb%?!N0)#bxpjRh<@alQ*!Iz@>eEji&a8~a#t2J3RN z=+2gSvDpZ3H9~9f;+{+A+{&prXJS)+38#n7`7k7ld;}RaJ`-$`g=@ae#_eW`DBGe= z)D)zT{u8Ln>^)H<^Rg|E-4+X=BdEbviud2}3V@uNQ}#_153W$;Cdts}(lq6KjTs1g9!B0Y=S5R-eSQ@l6q>~F1r4FYif9(lB92eJ91y& zjUl0iw+%|H;WZ0H%^q1=LgjdaJ)N?$%K~IKF?S0^0!MX+Qfg$a9Ql9Y^P2HDf71IJ zlGVv~!s~f)hEoGd*yl_?y{cs7JZI6NDup1ShO>GrxuD5MevV3MxFiYd&ytND6Ig@B zML3v#(`=*GQX?HXaLyqSdTKmuGm5OAQbrBXn!4~i2z;rk?*$U6Io4mvTet&L2@BAV z@N17~(~U*lIqU$y&R)$nhCaP1ZNs=0?o4aYGA_CPQY>josbLb_j&$hE+vuBtL~2-k z=Khmb*ZBQkY5HHrX^6rn5wcJgL7%s5+t5=ZZQRPU-tjOF&}xk^vAlgoPt8q4RhoV7 zYqK>XHLUARaUuKMY~wf!uMctWCOs7hP(vc*&n=ssbkt;oxM+E*6KUf(YlN+`?jhPjUMxW8Q{51vSAwfDD74O zEj5bGP{K2=FQrkd!Jc+KH9V42EL?bHS-Yi4q=pNY7qNMO5!hnb)I_{C*(iYHuy9)r zO|dD%0}3rQim3FKPNbF^p|$KvM-6|D^HEw;kE=D^2(2kYtwCO!tS$R9+(B!jcC*!C z(3V3Evw^Cn-Nd2yHNKP@9{n<3qosx#bgkC#$ts>iJlx#ZNMrmEA~lle9Rc|ow8zgX z;CUWncVppz>#0_&*$+f9;BrW7&mtnUmh6&qmk|+0XkXHrEvbl*%qXmihB zUe^G1T^Z~t^bT~5NRAeaJXjOiKmA!K`nHXpEUA0LGf-h;=KiPwaWYu6%gSW~{l^R< zHDJvw3ot;MVZDVCS}h?}{6bFC(jNwZQV5zdT7ZZgn#9Uqvy3|1r*s&t$ul!4O&Bp} z?#j9{Ib=CBmcWz+5N-PCy`$afw7vO}aPufiWEjE&Y@LWR(IPr(B;oTdA9*;BhoJ`k zlDGqyR}3Qdq8rYy_W{z>{Z|fZPSriyD2`$r}lwWeH`xhtP`J z#ko}mBv}KjdoIGK*+jBUxSm(*Rb&|XcxPwwo7OjWXJi*=2yXrOiIl;Y*sW4 zw|!Z??UDOy0lD}jGx1PUdZ4Z7N2Yko`W8C{x9Z-^_}&uuuh3y1N^4*owi<`&lO9VF zVzIA-2S?t$IY7G?WP)+S3OQIqh^}Y|P0iIkKFcop2*LP0&*B?*!dI)mh6$I*)k!7CNELrO*X>OTs7_aCm|wsrL|m{aWlwG9?+m21$7z4 z^C6xh1q>QvO}#6v?8z;+;!qgkPJ~yTjPYqgtjT8lNnT;I<*D^idpWZuLGMlBblbyH zbJR*nY>#Q=#A*^-2xWNSrrFDRU6Qcw8a>4^L*(HXp8W|nTbm}EAw8Q@8d^(mY?y2K zT`4ue?HcgmGL2q~3dx!w;O2C14TJXgorOQiU3JyVtf8no)(yi$5tV23qzCJTn*f2F za`c@l{HAqHA%tFtU2XZe_+x3k?2{jtLM`X)}e}|vL+s-qp+*Z*)Jzv>h2YtiFynmg;{gD_c#706*G>AJ9nrw#f%(MNy9yOt63lNhfyxflYqp!U!Is!Q_%-Sc^ zC{ZLmn?g-_lhSea6Pj(%9EDl?STCdbBOu$J(!!|@tEyPq5ktDXq4Dv9Z3$KC z1m6OZkrCfYw~)MEHNm%l7M!W^>hB?ImLMLYnrssM97u&g=u=PdbD+zc-}642KM$co zkhqs$gYkU3Eolk9A8*>%2`2bSOmTmvvq-r4HFzdHn@Miw_ki@%ZnNbq!8d{QY}(24 z=J$Z~Y-(P8lHi*`wxkLDwh4X?L(-?QznXJ`U4i!yB$2>x!VBakv1q>vu*oLTn%@Q9 zvl)5TS%Pl{Ti!$;+mzs&@w)Oi*3O&H66_xzHG7Gz4%0~R%^2R_-zS;p@4$bTs-Q6( SbiEA#0000 + +An expansion module to query ipinfo.io to gather more information on a given IP address. +- **features**: +>The module takes an IP address attribute as input and queries the ipinfo.io API. +>The geolocation information on the IP address is always returned. +> +>Depending on the subscription plan, the API returns different pieces of information then: +>- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field. +>- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse. +> +>More information on the responses content is available in the [documentation](https://ipinfo.io/developers). +- **input**: +>IP address attribute. +- **output**: +>Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s). +- **references**: +>https://ipinfo.io/developers +- **requirements**: +>An ipinfo.io token + +----- + #### [ipqs_fraud_and_risk_scoring](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py) diff --git a/documentation/website/expansion/ipinfo.json b/documentation/website/expansion/ipinfo.json new file mode 100644 index 0000000..070b7a8 --- /dev/null +++ b/documentation/website/expansion/ipinfo.json @@ -0,0 +1,13 @@ +{ + "description": "An expansion module to query ipinfo.io to gather more information on a given IP address.", + "logo": "ipinfo.png", + "requirements": [ + "An ipinfo.io token" + ], + "input": "IP address attribute.", + "output": "Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s).", + "references": [ + "https://ipinfo.io/developers" + ], + "features": "The module takes an IP address attribute as input and queries the ipinfo.io API. \nThe geolocation information on the IP address is always returned.\n\nDepending on the subscription plan, the API returns different pieces of information then:\n- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field.\n- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse.\n\nMore information on the responses content is available in the [documentation](https://ipinfo.io/developers)." +} From 113a112001b987956394f172ccb5c11472dccf7e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 2 Apr 2023 10:11:24 +0200 Subject: [PATCH 3/4] fix: [dbl_spamhaus] if you want to run local test, the dns module expansion is taking over from the original dnspython3 library. The trick is just to get rid of the syspath to exclude the local directory until the proper library is loaded. --- misp_modules/modules/expansion/dbl_spamhaus.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misp_modules/modules/expansion/dbl_spamhaus.py b/misp_modules/modules/expansion/dbl_spamhaus.py index 0cccfaf..4ecdfb9 100644 --- a/misp_modules/modules/expansion/dbl_spamhaus.py +++ b/misp_modules/modules/expansion/dbl_spamhaus.py @@ -2,7 +2,10 @@ import json import sys try: + original_path = sys.path + sys.path = original_path[1:] import dns.resolver + sys.path = original_path resolver = dns.resolver.Resolver() resolver.timeout = 0.2 resolver.lifetime = 0.2 From c1168ac627ee06ec352d632bf6f298b4d68d5de5 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 2 Apr 2023 10:47:41 +0200 Subject: [PATCH 4/4] fix: [test] pdftotext output check The important part is the matching text from the PDF not any trailling which might be different depending of the encoding. --- tests/test_expansions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 914ac2c..5f4d326 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -195,7 +195,7 @@ class TestExpansions(unittest.TestCase): query = {"module": "dbl_spamhaus", "domain": "totalmateria.net"} response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'totalmateria.net - spam domain') + self.assertEqual(self.get_values(response), 'totalmateria.net - spam test domain') except Exception: try: self.assertTrue(self.get_values(response).startswith('None of DNS query names exist:')) @@ -263,7 +263,7 @@ class TestExpansions(unittest.TestCase): self.assertEqual(to_check, 'OK (Not Found)', response) else: self.assertEqual(self.get_errors(response), 'Have I Been Pwned authentication is incomplete (no API key)') - + def test_hyasinsight(self): module_name = "hyasinsight" query = {"module": module_name, @@ -432,7 +432,7 @@ class TestExpansions(unittest.TestCase): encoded = b64encode(f.read()).decode() query = {"module": "pdf_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'Pdf test') + self.assertRegex(self.get_values(response), r'^Pdf test') def test_pptx(self): filename = 'test.pptx'