mirror of https://github.com/CIRCL/AIL-framework
38 lines
957 B
Python
Executable File
38 lines
957 B
Python
Executable File
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
##################################
|
|
# Import Project packages
|
|
##################################
|
|
from modules.abstract_module import AbstractModule
|
|
from lib.objects.Domains import Domain
|
|
from lib.objects.Items import Item
|
|
# from lib.ConfigLoader import ConfigLoader
|
|
|
|
class Languages(AbstractModule):
|
|
"""
|
|
Languages module for AIL framework
|
|
"""
|
|
|
|
def __init__(self):
|
|
super(Languages, self).__init__()
|
|
|
|
# Send module state to logs
|
|
self.logger.info(f'Module {self.module_name} initialized')
|
|
|
|
def compute(self, message):
|
|
item = Item(message)
|
|
if item.is_crawled():
|
|
domain = Domain(item.get_domain())
|
|
for lang in item.get_languages(min_probability=0.8):
|
|
domain.add_language(lang.language)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
module = Languages()
|
|
module.run()
|