From c26341c225054a3f868fddcce2089489631a30be Mon Sep 17 00:00:00 2001
From: airkeyp <romain.kieffer@sfr.fr>
Date: Thu, 26 Sep 2019 11:56:04 +0200
Subject: [PATCH] chg: [export] Export script added

---
 .gitignore                                |   4 ++-
 bin/export.py                             |  29 ++++++++++++++++++----
 install.sh                                |   2 +-
 lib/__pycache__/inspection.cpython-36.pyc | Bin 7275 -> 0 bytes
 lib/analyzer.py                           |  20 +++++++--------
 lib/inspection.py                         |  12 +++++----
 6 files changed, 44 insertions(+), 23 deletions(-)
 delete mode 100644 lib/__pycache__/inspection.cpython-36.pyc

diff --git a/.gitignore b/.gitignore
index 0b87200..e0887e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 .idea/
 ipa.egg-info/
 .venv
-db/
\ No newline at end of file
+exports/
+db/
+__pycache__/
\ No newline at end of file
diff --git a/bin/export.py b/bin/export.py
index b3fbb35..714e905 100644
--- a/bin/export.py
+++ b/bin/export.py
@@ -21,21 +21,40 @@
 import markdown_strings as mds
 import redis
 import os
-
+import time
 
 analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1')
 analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6405))
 r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port)
 
+table_line = '| :--------------- | :--------------- |\n'
+padding = [16, 16]
+
+
+def init_export_dir(path: str):
+    if not os.path.exists(path):
+        os.mkdir(path)
+
 
 def export_icmp_types():
-    res = mds.table_row(['ICMP Type', 'Count'], [10, 10]) + '\n'
-    res += '| :----- | -----: |\n'
+    res = mds.table_row(['ICMP Type', 'Count'], padding) + '\n' + table_line
     redis_dict = r.hgetall('icmp')
     for key in redis_dict:
-        res += mds.table_row([key.decode(), redis_dict[key].decode()], [10, 10]) + '\n'
+        res += mds.table_row([key.decode(), redis_dict[key].decode()], padding) + '\n'
+    return res
+
+
+def export_protocols():
+    res = mds.table_row(['Protocol', 'Count'], padding) + '\n' + table_line
+    redis_list = r.zrange('protocols', 0, -1, withscores=True)
+    for item in redis_list:
+        res += mds.table_row([item[0].decode(), int(item[1])], padding) + '\n'
     return res
 
 
 if __name__ == "__main__":
-    export_icmp_types()
+    pwd = os.getcwd() + '/exports/'
+    init_export_dir(pwd)
+    with open(pwd + str(time.time())[:10] + '-export.md', 'w') as exp_file:
+        exp_file.write(export_icmp_types() + '\n')
+        exp_file.write(export_protocols() + '\n')
diff --git a/install.sh b/install.sh
index e1df4aa..0359d33 100755
--- a/install.sh
+++ b/install.sh
@@ -7,6 +7,6 @@ export PIPENV_VENV_IN_PROJECT=1
 
 if [ -z "$VIRTUAL_ENV" ]; then
     pipenv install
-    export IPA_HOME=$(pwd)
+    echo export IPA_HOME=$(pwd) >> .venv/bin/activate
 fi
 
diff --git a/lib/__pycache__/inspection.cpython-36.pyc b/lib/__pycache__/inspection.cpython-36.pyc
deleted file mode 100644
index 48257e87f3c7929d9485f9835fd429d98fe87c4b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 7275
zcmcIp-ESP%b)T=@9WIwkiXufyaqLy<*jhzPiQkf~sQyr##BmrWQfg;xWj&m8xg2tK
zW_V{-BB#S9E#)R{;>0QP&<CSH`xYQ5`qG#7rO$mR`dAE7pg>We`WFlo?eE-KE+s|}
z4T3B&_w(F)&pr2d&&TER#6<mXfBwCD&poSY|EWFpTSxg7e93?4nnoJwf!5a@UFWh9
z7=6PrRM`wn$3&YISpAAqL0g5YRO^{!)7V2zRH;tmN5*aKNT1W38a0Nk6VxnQ9h)X;
ziq6n9ou!#xm1c(&O*%IkH%4<bPv_|ZeTJT)&(gE>9JS~oeU5&Qo~O^#3-lsgqBdQo
zFVGe0&;nhhYqUs9v`j0sN^7)E*FU-F)ak}>_Lq*ia>nUpx=CN8TlD+%3cX6N(d%@Z
zzC>@(m+33?ReF=YMsLyAX@kB&Z__vFTXcu+(zodk=pA~OzC#YZNAJ@;`Yv_p19EAT
zJR%abMLpUkpLS@M0_syp5yiCk$(qw3Ny$UanGj7%KhbGG_a9o$q?i&-v^@C4q<uO-
z%Na368T$W_a@3~jkp2jx&eHej`^abLkLgd4&(aY+M1GDwq(4PIM?at+BA=%}qd!M}
zo_<6>Mt*@lqMsoDjL_+)nDt}&8D@Qk%sX29FCOvHxAkKD4L=aCyKy>@0#&OQM4KBZ
zO!h?DO(Jg>S?s3Ug}zwmONGAf0UqjSAhqzljxU)(($jM7UgJ<twOl{ae`f4iceG#d
z8fKEpwow=h*9zU2Hb_<S>8-2VQD0p3eCY*OZ%5vsFTymrN-Lc(3OlJ6cQ##bH}Tvw
z74mA}Z(jAoBo<!kN8v(z@M-;K;3ugU_2XMSX~G;;H93Lo(G^bq2gz^7uVSpN%Q1{>
z8?uf}j&m~NS~j@KYcS1l%}2cdN4Qd2p|2GBT45}%;^TZB-}T=e6U^Z;6PHlVY~gK3
z9Vy~$ibRrj2B8$Lx9x5QBAZB}f%HUYZy-W%yRfAoUqYVQv>QOgp__`%c7#DLky9Up
zZj$)DP>^Hsq}hy&1|UQ4eIZjn5qw_R*<ouE1-|E}E}u>|p89>!5&NDHl-aQ>-F_$D
zEXKGwN!@;&O^k|)*@ho(G237*NwfN}76%8}6qu1BNjiNu*=34fYl)qDDqT+~MzWcH
zv<Xa?8yW5JJh`dcV>Y}f1p5iN><B3%iN)gkYn=z%LW*oml~wWVa2_doKpD2pOFyMC
z&Z-ou&N7b+E7|p9$Bwrn+$ulGX2pIiBz7LAZqQNF`sqP-M(n2|gyQH7k8JPPo8oLt
zZh6aVH<mh!C&i`q#X`qrT9(raa;x$za#>TA>xF&;n7f8=5#N$yEH2|)k&IC}%gH%T
z=8!nX@-=**TVBGqjBf??ReWpsfZgTm$Zt5t3g%k@-3sPgS$52&Rh6v#?vQ;I{2e3v
zr_5`0JBp(?EiB*b$ByOq`f*`GOE{+J#o74gCQiZgAm7lj_Ct|66~I193Uff9=@0g^
zS}F(OE+jcDEM?U|xLch?$CRM4e3`~s%}=6E8n0wyeH>ca@k8{G$LlP|9uPvY?0Ef@
z=>Z(aN=TxwCj}TuV#jc|9V>LvRxJ_aSZR>pAQiNj_aWk-<n~kNf*T%m`O<a$P~E|x
z8_Ot-yeM!iZwt$uVTvH~+@OnHJ&5EkuMAddiJO+&z3(S3sN-<oG2{DKU{iL6omut4
zW=Z77X+Nun$rfhm_&xCN?k}x6HIV}E0Bi+H!R1z9)-m_gT)n<y1xsqLE>bh$ZTX3x
z++WEikCEKj7A^@~q<$PEkXOf#+iUU}23duZDkln|&vEG@Cr<_A5`ek{0D>!HX&oQH
zw{$~3&jUWs$qSsk$O-E@*+v35f^rowUR}h8(W@A}3iz&K^eUjchW0hUb`7(yfp!gT
z>!4pp+d631LAwsx>lk|-^&6nO0lFL43v=<>3MXqw7MD3$<z$_c8%UP8cbVIkPYfd)
zNZztCj8iDUAi|#_1Apd6Fp6z!Lw*76ciWZ&O&1T+?iMtb(<DEM1NVSyfP!NW64BjB
zfah^P^nwA2u5=&3I<cW^>&`eE=588wacrjyR8eF5jrJXos#~P#l3hHQ<Ej+UOW_kY
zmuFP`vUOvAkMu)5*TJivy<BhM_hpeLEg+@Ei?*Vz7Esw@wp)Iv+I#+e5w=Q$63K%F
zH44m{ysA-H%8RhTQgWKtG3J%0N<z>k>{Mu$t;?5rq**jrbtlJuNR8YBHv95AdOhZw
ze=P5jGx7#1J?+rQ^`z0$?r8ffxqe%_cjeH`d2c3rriAa%O7+~rPAhq3$2ih-EjRMY
zI!5kIzk@P%Z)ESQeY_4Mnu?_5ws`H<10OPQgFtPZx750%+94LO2RTI$lT1YWlyF>#
zEDCT}l2SZb<?7{yUe-31!=W||>k-McU6nVn3;9JPj*S(%Jbz(g_Oi;-lrJ&Klb%U2
z;fKCDc{SobC=xajnq}$ySJ$U>8-ER5zJgZ3p#poHijgGw#*QSpgi0yNof5<5k$GtD
zU&u|!bNbNAjU7mkWp{)~VQ+pYHQ<{s!6TLYFMe&)sRq&wd!neiaSW?hR7rRdiFQT4
z3euv&mO4>8a;lr<ZY@6*{nPs!V?i*olMgX2VgE<NzM9zCv@XAb+6dT;ywO(PMuqfT
z+tGIPT}U@IR2g7^$7-?1npXIN9vSFe%5}2l0G+g&>*oQ8(4xv6__b7>(=gIt2mv0o
zjl%ZhQen%>SV2xA$tGHtPMoK9HpA}hW3FsR83D8PQmb84il7+Z@<l-1(1mwYLgA~J
zr!Zj>AQ>gMy0Qvg^1T#yq8h_97HMH0+X~48PRpvn7weefBM>A_BpMFC0b$vC-LQ1Y
zf*Fa7PmBSjbipZ+&7X)2qELclfe}Q*5g^&U+O8?zf-qGdBcd#GiN@|Al7z_W8->Le
zy0FrNSQHk6ps>Ad0Z)I>FD8#mU54c#DJnj}Z$b<q_M9q`F8tWqMudBox&ROBQTZk(
z?2gNCA}K1dgy5ihHn{23{g`5z@BtvKIS3+`{t`q9tG!Y#7a*IuF294i#{l9V=OYdI
zJ}Ll$zH7+KIdnSm`EtjQ3gm(MyQ&WT5iT&0&~kT3`I_oM1oi|aY(fbzC86qu<md%k
zmL6_AR9y#Eb)$wGcE;iMbkQ9{Rg@Kwl@&NUCp}yl_CUudaeM=FIW}%!96?B@rDH3^
zVAj6|S;Bszrql(ijQUB*kSv7}YFIX-i^i7y*n&X6Dp?(I%xP>$2Ym>u7+7isl$j|$
zF}OUZnNje?F1x(5zXM%q8o-A9E^5E|^6z2PQ<j(a(EilLAv?AB-#@W<_w?c;Y_!mG
zymzH+^c_|w&*0Sc9mqhPudz{7p)|0uwpEJa1jHX;f}v2J%GLM?y?*=;piS8D@ex6v
zyC{uh!l-B1s*4<;(8Hgh9qNb1KHv%U{q~_*8bhdY6YLqewPhSuQj-njVO6zRc@=H7
zyprp?HMx=NJ1~`eo%335pk)dMppw^=@}Gk8*K=cw&8EHq1IA}m-p>!X#8QLrr~oN%
zqtdnu6MDR;l~HNp)KXVse~H3`(>y-;ZaRnqxI;X?d3++nMn2KgCpWJw#BkEC8&Y>T
z?>~Wfi0QgQb3+NPM0Qh`g#M8LIhbNw$U;nhj)Y(+exPl!A7?(aQ`A%ahp>BO9l;tx
zWJYR0cm`F@Yg9dgi8wR~Vxr1TZU2Q-!(Bkz5ql!%Aab2)bE~9<&!O+V`3~OFAY$uC
zyM@xchQojllUH8l*{eB@1e)Z>WCKSh+EnR<5!Z}Fh6H=t347q`QD){4oMI-Ov^niV
z<GfZr&b0MMd~22b06I3M>ahcwll5pRSJsT`x#M3^7{+7Si%LtEYV=rPFWgiyN^%PY
zm}vDZBL$Cs4auV^B#w0rKga4t5jp0jOVDR;p}jBz5f<jQ*q2}9YJC$f44QiBHY_Cb
z;?&UDg@JJDaFNl=>R;YPkK(wn_;H{x@lH_gX1D`pzBv07X8#2pLnv7Kv&tAw>*wGb
zF(fQQ@{Jh@^$GRQF44%3sL5lah=awg&~{At7ZzM+Hh~sgP#oZ{awLb!uTmI(T#R?&
zJi_IL(NE;Nn6bPkN<Ef(+G!{oh3mr4A5LvpIF0p*a&JSh|2yxh^oZERv%;1U+Hjzo
z$VNV16BR~@QV#zOo6qsdt}SPPYG4%B8oCN)_{P|jzM~((1^QUu|0OGqkM(<h15euA
zF_hvkfl>?pGt|OiRh6nJ)zX?$BX(+oa%`uLwz1r%3K0BP>A32vLwPhx&%xJejOrhm
z?16JzUMKscjX)2<PC%3XDsA#u`71@&V6WqZu544Z#?B=79Pyj=gld`Et$vKiDLtds
z9tZ6-HS=+JuaigRx@Yk&UE>jXHJwq^u<tu)&8svur*Ths34ODw?~PI4nPFe!xJAvu
zBd3S+sd7HX?WvWXV-BD<&dzDMjYx!ZxRoleAu8c{8Z-lGJ+`-WPVqgF&gT;tJ6q<j
z!R(saF3u+UXF9hFH`PECBd_dTVNZPLJjXOR$GPJ(ZO>=dR6KLrPZB)*l@T43a2d%V
zY{8opp3+jpEn86;qikfe@4b7k1(%DR?onuSQ7NA6+<TYrweH|9rh-(=<A=S4g@w#s
zgnP>n9=c>{gtsrH3GO0twGhur*#`uj2`CCY+2F02pC;u8Xp<f%Y!pv>(r~Ov#CWnq
z#6oR#|3B4Va>kB*+~g6v7XCj1(%C$7_O>6301;8xW5!jCvCzJtt{i;lvQsEV${VPh
z$ZzrTcR1nMlrC^=bsZ6Y7nS>NFc8W?bgYbTXUJ51&y<ZSy-_yCj<t=j6{?GOyLnK|
zo&*J@>SPZ%DP~W$^jLFgKG|q0orq_eec|ExwcS*AS#ppo6HeHJ0)Ge*-2<4(y5h7f
zkEt>*<CI1Awk5lO^4my?+Rz795-9KhluE>x$2Rt~vKdn`EyrH}g~180%W*m@Oxv&#
zf>rgZF|@3x+xm<?W3WP<(a#x8<Ws0|n1;`2pxiK?H)l*+e_=R-Jiv_Y&pNgEAlY{1
zu6pQD4+wCE4-#h_{{vXq76D#U6RAuWTMs!v;>^lubC9GVUWy-=A_u@ub0h{8KRHv6
z9YpDQex2vx_mRS4<E0|;GNO2edpN8t!N75Wa$O%gA3kg4JSeD?NZ_43CO;K~C~IWj
z>_;>R#4TV{Oa2WBt7_Y5n9o8^&l`sR+x^+*%x^u%|DOZfvaO~)J=wJFaeK_3v@7_l
J+D(+~{{#6dz&`*0

diff --git a/lib/analyzer.py b/lib/analyzer.py
index 9dd20b1..8d09076 100644
--- a/lib/analyzer.py
+++ b/lib/analyzer.py
@@ -24,7 +24,7 @@ import time
 import configparser
 import logging
 
-from lib.inspection import get_cap, get_protocol, check_icmp_checksum, get_icmp_payload, get_icmp_ip, \
+from lib.inspection import get_cap, get_raw_cap, get_protocol, check_icmp_checksum, get_icmp_payload, get_icmp_ip, \
     unassigned_icmp_types, deprecated_icmp_types, get_src_port, get_dst_port, list_caps, init_cap_list
 
 
@@ -69,7 +69,7 @@ class Analyzer:
             self.cap_list = []
             self.logger.info("Adding dataset caps to local queue")
             self.cap_list = init_cap_list(self.dataset)
-            self.logger.info(len(self.cap_list))
+            self.logger.info('Added ' + str(len(self.cap_list)) + ' caps.')
             self.update_queue()
             self.logger.info("Processing...")
             self.process_local()
@@ -79,7 +79,6 @@ class Analyzer:
             if c == 0:
                 self.enqueue_caps(cap_list=list_caps('scanning', self.r))
                 self.r.delete('scanning')
-                print('[-] Process remaining unfinished caps.')
                 self.process_local()
 
     def enqueue_caps(self, cap_list: list):
@@ -105,7 +104,6 @@ class Analyzer:
             self.logger.info('Queue updated.')
         else:
             if self.cap_list:
-                self.logger.info('No caps enqueued, initializing...')
                 caps_to_add = self.cap_list
             elif current_caps:
                 return 0
@@ -117,7 +115,7 @@ class Analyzer:
         Dissects the cap file to extract info.
         """
         if cap is None:
-            self.logger.info('[X] No caps to parse!')
+            self.logger.info('No caps to parse!')
             return 0
 
         self.logger.info('Parsing cap ' + cap.input_filename[-15:])
@@ -130,7 +128,7 @@ class Analyzer:
             icmp_type = str(icmp_layer.type)
             # icmp_code = str(icmp_layer.code)
             protocol = get_protocol(packet)
-            checksum_status = check_icmp_checksum(packet.icmp_raw.value)
+            # checksum_status = check_icmp_checksum(packet.icmp_raw.value)
 
             if protocol == '1 : icmp':
                 payload = get_icmp_payload(packet)
@@ -149,8 +147,8 @@ class Analyzer:
             else:
                 pipeline.hincrby('icmp', icmp_type)
 
-            pipeline.hincrby('checksum', 'total')
-            pipeline.hincrby('checksum', checksum_status)
+            # pipeline.hincrby('checksum', 'total')
+            # pipeline.hincrby('checksum', checksum_status)
 
             # entry = str(get_src_port(packet)) + ':' + protocol + ':' + icmp_type + ':' + icmp_code
             # pipeline.zadd(source_ip, {entry: 1}, incr=True)
@@ -172,11 +170,11 @@ class Analyzer:
             absolute_path = self.r_d4.rpop(self.queue).decode()
         else:
             absolute_path = self.r.lpop('to_scan').decode()
-        return get_cap(absolute_path)
+        return get_cap(absolute_path), get_raw_cap(absolute_path)
 
     def process_d4(self):
         while True:
-            d4_cap = self.pop_cap()
+            d4_cap, d4_raw_cap = self.pop_cap()
             if d4_cap is None:
                 time.sleep(1)
                 continue
@@ -186,7 +184,7 @@ class Analyzer:
 
     def process_local(self):
         while self.r.llen(self.queue) != 0:
-            cap = self.pop_cap()
+            cap, raw_cap = self.pop_cap()
             self.r.rpush('scanning', cap.input_filename)
             self.parse_cap(cap)
             self.r.lrem('scanning', 0, cap.input_filename)
diff --git a/lib/inspection.py b/lib/inspection.py
index d7b4a47..cac07a1 100644
--- a/lib/inspection.py
+++ b/lib/inspection.py
@@ -120,10 +120,14 @@ proto_dict = {
     }
 
 
-def get_cap(path_to_cap):
+def get_raw_cap(path_to_cap: str):
     return FileCapture(input_file=path_to_cap, display_filter='icmp', use_json=True, include_raw=True)
 
 
+def get_cap(path_to_cap: str):
+    return FileCapture(input_file=path_to_cap, display_filter='icmp')
+
+
 def get_files(path) -> list:
     caps = glob(path)
     return caps
@@ -151,11 +155,11 @@ def list_caps(state: str, redis):
 def get_protocol(packet):
     if 'ip_proto' in packet.icmp.field_names:
         protocol = str(packet.icmp.ip_proto)
-        if int(protocol) in range(143, 253):
+        if protocol in unassigned_proto:
             return protocol + ' (unassigned)'
         ip_proto = proto_dict[protocol]
     else:
-        return 'non-backscatter-icmp'
+        return 'nbs-icmp'
     return protocol + ' : ' + str(ip_proto)
 
 
@@ -163,8 +167,6 @@ def get_icmp_payload(packet):
     if 'data' in packet.icmp.field_names:
         return str(packet.icmp.data)
     elif packet.icmp.field_names != ['type', 'code', 'checksum', 'checksum_status', 'ident', 'seq', 'seq_le']:
-        print(packet.icmp.field_names)
-        print(packet.icmp)
         return 'No data'