mirror of https://github.com/MISP/misp-galaxy
Refactor [generator] generate mkdocs site
niclas
parent
262b95fa79
commit
b8b24f74ec
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
from typing import List
|
||||||
|
|
||||||
import validators
|
import validators
|
||||||
|
|
||||||
|
|
@ -16,9 +17,9 @@ for f in os.listdir(pathClusters):
|
||||||
galaxies_fnames.append(f)
|
galaxies_fnames.append(f)
|
||||||
|
|
||||||
galaxies_fnames.sort()
|
galaxies_fnames.sort()
|
||||||
|
galaxy_output = {}
|
||||||
|
|
||||||
index_output = ""
|
intro = """
|
||||||
index_output += """
|
|
||||||
# MISP Galaxy
|
# MISP Galaxy
|
||||||
|
|
||||||
The MISP galaxy offers a streamlined approach for representing large entities, known as clusters, which can be linked to MISP events or attributes. Each cluster consists of one or more elements, represented as key-value pairs. MISP galaxy comes with a default knowledge base, encompassing areas like Threat Actors, Tools, Ransomware, and ATT&CK matrices. However, users have the flexibility to modify, update, replace, or share these elements according to their needs.
|
The MISP galaxy offers a streamlined approach for representing large entities, known as clusters, which can be linked to MISP events or attributes. Each cluster consists of one or more elements, represented as key-value pairs. MISP galaxy comes with a default knowledge base, encompassing areas like Threat Actors, Tools, Ransomware, and ATT&CK matrices. However, users have the flexibility to modify, update, replace, or share these elements according to their needs.
|
||||||
|
|
@ -36,7 +37,7 @@ Clusters serve as an open and freely accessible knowledge base, which can be uti
|
||||||
## Publicly available clusters
|
## Publicly available clusters
|
||||||
|
|
||||||
"""
|
"""
|
||||||
index_contributing = """
|
contributing = """
|
||||||
|
|
||||||
# Contributing
|
# Contributing
|
||||||
|
|
||||||
|
|
@ -46,89 +47,181 @@ We encourage collaboration and contributions to the [MISP Galaxy JSON files](htt
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
galaxy_output = {}
|
class Galaxy():
|
||||||
|
def __init__(self, cluster_list: List[dict], authors, description, name, json_file_name):
|
||||||
|
self.cluster_list = cluster_list
|
||||||
|
self.authors = authors
|
||||||
|
self.description = description
|
||||||
|
self.name = name
|
||||||
|
self.json_file_name = json_file_name
|
||||||
|
self.clusters = self._create_clusters()
|
||||||
|
self.entry = ""
|
||||||
|
|
||||||
for f in galaxies_fnames:
|
def _create_metadata_entry(self):
|
||||||
with open(os.path.join(pathClusters, f)) as fr:
|
self.entry += "---\n"
|
||||||
cluster = json.load(fr)
|
self.entry += f'title: {self.name}\n'
|
||||||
cluster_filename = f.split('.')[0]
|
meta_description = self.description.replace("\"", "-")
|
||||||
index_output += f'- [{cluster["name"]}](./{cluster_filename}/index.md)\n'
|
self.entry += f'description: {meta_description}\n'
|
||||||
galaxy_output[cluster_filename] = "---\n"
|
self.entry += "---\n"
|
||||||
galaxy_output[cluster_filename] += f'title: {cluster["name"]}\n'
|
|
||||||
meta_description = cluster["description"].replace("\"", "-")
|
|
||||||
galaxy_output[cluster_filename] += f'description: {meta_description}\n'
|
|
||||||
galaxy_output[cluster_filename] += "---\n"
|
|
||||||
galaxy_output[cluster_filename] += f'# {cluster["name"]}\n'
|
|
||||||
galaxy_output[cluster_filename] += f'{cluster["description"]}\n'
|
|
||||||
|
|
||||||
if 'authors' in cluster:
|
def _create_title_entry(self):
|
||||||
if cluster["authors"]:
|
self.entry += f'# {self.name}\n'
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
|
||||||
galaxy_output[cluster_filename] += f'??? info "Authors"\n'
|
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
|
||||||
galaxy_output[cluster_filename] += f' | Authors and/or Contributors|\n'
|
|
||||||
galaxy_output[cluster_filename] += f' |----------------------------|\n'
|
|
||||||
for author in cluster["authors"]:
|
|
||||||
galaxy_output[cluster_filename] += f' |{author}|\n'
|
|
||||||
|
|
||||||
for value in cluster["values"]:
|
def _create_description_entry(self):
|
||||||
galaxy_output[cluster_filename] += f'## {value["value"]}\n'
|
self.entry += f'{self.description}\n'
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
|
||||||
if 'description' in value:
|
|
||||||
galaxy_output[cluster_filename] += f'{value["description"]}\n'
|
|
||||||
|
|
||||||
if 'meta' in value:
|
def _create_authors_entry(self):
|
||||||
if 'synonyms' in value['meta']:
|
if self.authors:
|
||||||
if value['meta']['synonyms']: # some cluster have an empty list of synomyms
|
self.entry += f'\n'
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
self.entry += f'??? info "Authors"\n'
|
||||||
galaxy_output[cluster_filename] += f'??? info "Synonyms"\n'
|
self.entry += f'\n'
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
self.entry += f' | Authors and/or Contributors|\n'
|
||||||
galaxy_output[cluster_filename] += f' "synonyms" in the meta part typically refer to alternate names or labels that are associated with a particular {cluster["name"]}.\n\n'
|
self.entry += f' |----------------------------|\n'
|
||||||
galaxy_output[cluster_filename] += f' | Known Synonyms |\n'
|
for author in self.authors:
|
||||||
galaxy_output[cluster_filename] += f' |---------------------|\n'
|
self.entry += f' |{author}|\n'
|
||||||
for synonym in sorted(value['meta']['synonyms']):
|
|
||||||
galaxy_output[cluster_filename] += f' | `{synonym}` |\n'
|
|
||||||
|
|
||||||
if 'uuid' in value:
|
def _create_clusters(self):
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
clusters = []
|
||||||
galaxy_output[cluster_filename] += f'??? tip "Internal MISP references"\n'
|
for cluster in self.cluster_list:
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
clusters.append(Cluster(
|
||||||
galaxy_output[cluster_filename] += f' UUID `{value["uuid"]}` which can be used as unique global reference for `{value["value"]}` in MISP communities and other software using the MISP galaxy\n'
|
value=cluster.get('value', None),
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
description=cluster.get('description', None),
|
||||||
|
uuid=cluster.get('uuid', None),
|
||||||
|
date=cluster.get('date', None),
|
||||||
|
related=cluster.get('related', None),
|
||||||
|
meta=cluster.get('meta', None)
|
||||||
|
))
|
||||||
|
return clusters
|
||||||
|
|
||||||
if 'meta' in value:
|
def _create_clusters_entry(self):
|
||||||
if 'refs' in value['meta']:
|
for cluster in self.clusters:
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
self.entry += cluster.create_entry()
|
||||||
galaxy_output[cluster_filename] += f'??? info "External references"\n'
|
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
|
||||||
|
|
||||||
for ref in value["meta"]["refs"]:
|
def create_entry(self):
|
||||||
if validators.url(ref): # some ref are not actual URL (TODO: check galaxy cluster sources)
|
self._create_metadata_entry()
|
||||||
galaxy_output[cluster_filename] += f' - [{ref}]({ref}) - :material-archive: :material-arrow-right: [webarchive](https://web.archive.org/web/*/{ref})\n'
|
self._create_title_entry()
|
||||||
else:
|
self._create_description_entry()
|
||||||
galaxy_output[cluster_filename] += f' - {ref}\n'
|
self._create_authors_entry()
|
||||||
|
self._create_clusters_entry()
|
||||||
|
return self.entry
|
||||||
|
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
class Cluster():
|
||||||
|
def __init__(self, description, uuid, date, value, related, meta):
|
||||||
|
self.description = description
|
||||||
|
self.uuid = uuid
|
||||||
|
self.date = date
|
||||||
|
self.value = value
|
||||||
|
self.related = related
|
||||||
|
self.meta = meta
|
||||||
|
self.entry = ""
|
||||||
|
|
||||||
|
def _create_title_entry(self):
|
||||||
|
self.entry += f'## {self.value}\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
|
||||||
|
def _create_description_entry(self):
|
||||||
|
if self.description:
|
||||||
|
self.entry += f'{self.description}\n'
|
||||||
|
|
||||||
|
def _create_synonyms_entry(self):
|
||||||
|
if isinstance(self.meta, dict) and self.meta.get('synonyms'):
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f'??? info "Synonyms"\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f' "synonyms" in the meta part typically refer to alternate names or labels that are associated with a particular {self.value}.\n\n'
|
||||||
|
self.entry += f' | Known Synonyms |\n'
|
||||||
|
self.entry += f' |---------------------|\n'
|
||||||
|
for synonym in sorted(self.meta['synonyms']):
|
||||||
|
self.entry += f' | `{synonym}` |\n'
|
||||||
|
|
||||||
|
def _create_uuid_entry(self):
|
||||||
|
if self.uuid:
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f'??? tip "Internal MISP references"\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f' UUID `{self.uuid}` which can be used as unique global reference for `{self.value}` in MISP communities and other software using the MISP galaxy\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
|
||||||
|
def _create_refs_entry(self):
|
||||||
|
if isinstance(self.meta, dict) and self.meta.get('refs'):
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f'??? info "External references"\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
|
||||||
|
for ref in self.meta['refs']:
|
||||||
|
if validators.url(ref):
|
||||||
|
self.entry += f' - [{ref}]({ref}) - :material-archive: :material-arrow-right: [webarchive](https://web.archive.org/web/*/{ref})\n'
|
||||||
|
else:
|
||||||
|
self.entry += f' - {ref}\n'
|
||||||
|
|
||||||
|
self.entry += f'\n'
|
||||||
|
|
||||||
|
def _create_associated_metadata_entry(self):
|
||||||
|
if isinstance(self.meta, dict):
|
||||||
excluded_meta = ['synonyms', 'refs']
|
excluded_meta = ['synonyms', 'refs']
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
self.entry += f'\n'
|
||||||
galaxy_output[cluster_filename] += f'??? info "Associated metadata"\n'
|
self.entry += f'??? info "Associated metadata"\n'
|
||||||
galaxy_output[cluster_filename] += f'\n'
|
self.entry += f'\n'
|
||||||
galaxy_output[cluster_filename] += f' |Metadata key |Value|\n'
|
self.entry += f' |Metadata key |Value|\n'
|
||||||
galaxy_output[cluster_filename] += f' |---------------------|-----|\n'
|
self.entry += f' |------------------|-----|\n'
|
||||||
for meta in sorted(value['meta']):
|
for meta in sorted(self.meta.keys()):
|
||||||
if meta in excluded_meta:
|
if meta not in excluded_meta:
|
||||||
continue
|
self.entry += f' | {meta} | {self.meta[meta]} |\n'
|
||||||
galaxy_output[cluster_filename] += f' | `{meta}` |{value["meta"][meta]}|\n'
|
|
||||||
|
|
||||||
index_output += index_contributing
|
def _create_related_entry(self):
|
||||||
|
if self.related:
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f'??? info "Related clusters"\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
self.entry += f'```mermaid\n'
|
||||||
|
self.entry += f'graph TD\n'
|
||||||
|
for related in self.related:
|
||||||
|
self.entry += f' {self.value} --> {related}\n'
|
||||||
|
self.entry += f'```\n'
|
||||||
|
|
||||||
with open(os.path.join(pathSite, 'index.md'), "w") as index:
|
|
||||||
index.write(index_output)
|
|
||||||
|
|
||||||
for f in galaxies_fnames:
|
def create_entry(self):
|
||||||
cluster_filename = f.split('.')[0]
|
self._create_title_entry()
|
||||||
pathSiteCluster = os.path.join(pathSite, cluster_filename)
|
self._create_description_entry()
|
||||||
if not os.path.exists(pathSiteCluster):
|
self._create_synonyms_entry()
|
||||||
os.mkdir(pathSiteCluster)
|
self._create_uuid_entry()
|
||||||
with open(os.path.join(pathSiteCluster, 'index.md'), "w") as index:
|
self._create_refs_entry()
|
||||||
index.write(galaxy_output[cluster_filename])
|
self._create_associated_metadata_entry()
|
||||||
|
self._create_related_entry()
|
||||||
|
return self.entry
|
||||||
|
|
||||||
|
cluster_dict = {}
|
||||||
|
|
||||||
|
galaxies = []
|
||||||
|
for galaxy in galaxies_fnames:
|
||||||
|
with open(os.path.join(pathClusters, galaxy)) as fr:
|
||||||
|
galaxie_json = json.load(fr)
|
||||||
|
galaxies.append(Galaxy(galaxie_json['values'], galaxie_json['authors'], galaxie_json['description'], galaxie_json['name'], galaxy.split('.')[0]))
|
||||||
|
|
||||||
|
def create_index(intro, contributing, galaxies):
|
||||||
|
index_output = intro
|
||||||
|
for galaxie in galaxies:
|
||||||
|
index_output += f'- [{galaxie.name}](./{galaxie.json_file_name}/index.md)\n'
|
||||||
|
index_output += contributing
|
||||||
|
return index_output
|
||||||
|
|
||||||
|
def create_galaxies(galaxies):
|
||||||
|
galaxy_output = {}
|
||||||
|
for galaxie in galaxies:
|
||||||
|
galaxy_output[galaxie.json_file_name] = galaxie.create_entry()
|
||||||
|
return galaxy_output
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
index_output = create_index(intro, contributing, galaxies)
|
||||||
|
galaxy_output = create_galaxies(galaxies)
|
||||||
|
|
||||||
|
with open(os.path.join(pathSite, 'index.md'), "w") as index:
|
||||||
|
index.write(index_output)
|
||||||
|
|
||||||
|
for f in galaxies_fnames:
|
||||||
|
cluster_filename = f.split('.')[0]
|
||||||
|
pathSiteCluster = os.path.join(pathSite, cluster_filename)
|
||||||
|
if not os.path.exists(pathSiteCluster):
|
||||||
|
os.mkdir(pathSiteCluster)
|
||||||
|
with open(os.path.join(pathSiteCluster, 'index.md'), "w") as index:
|
||||||
|
index.write(galaxy_output[cluster_filename])
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue