fix: [Onion] filter too many subdomain

2019-02-13 16:53:38 +01:00 · 2019-02-13 16:53:38 +01:00 · 709d5487b8
parent b87707e8bc
commit 709d5487b8
1 changed files with 4 additions and 0 deletions
--- a/bin/Onion.py
+++ b/bin/Onion.py
@ -224,6 +224,10 @@ if __name__ == "__main__":
                            else:
                                continue

+                            # too many subdomain
+                            if len(domain.split('.')) > 5:
+                                continue
+
                            if not r_onion.sismember('month_onion_up:{}'.format(date_month), domain) and not r_onion.sismember('onion_down:'+date , domain):
                                if not r_onion.sismember('onion_domain_crawler_queue', domain):
                                    print('send to onion crawler')