CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Avoid indexing URLs multiple times

pull/184/head
Raphaël Vinot 2021-03-12 13:18:36 +01:00
parent aff9c62508
commit e90fc151bd
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lookyloo/indexing.py
View File

@ -170,8 +170,8 @@ class Indexing():
if self.redis.sismember('indexed_urls', crawled_tree.uuid): if self.redis.sismember('indexed_urls', crawled_tree.uuid):
# Do not reindex # Do not reindex
return return
self.redis.sadd('indexed_urls', crawled_tree.uuid)
pipeline = self.redis.pipeline() pipeline = self.redis.pipeline()
pipeline.sadd('indexed_urls', crawled_tree.uuid)
for urlnode in crawled_tree.root_hartree.url_tree.traverse(): for urlnode in crawled_tree.root_hartree.url_tree.traverse():
if not urlnode.hostname or not urlnode.name: if not urlnode.hostname or not urlnode.name:
continue continue
@ -203,7 +203,6 @@ class Indexing():
def index_categories_capture(self, capture_uuid: str, categories: Iterable[str]): def index_categories_capture(self, capture_uuid: str, categories: Iterable[str]):
if not categories: if not categories:
return return
print(capture_uuid, categories)
if self.redis.sismember('indexed_categories', capture_uuid): if self.redis.sismember('indexed_categories', capture_uuid):
# do not reindex # do not reindex
return return