Fix [tool] file creation

tools/mkdocs/main.py

@@ -15,15 +15,13 @@ CLUSTER_PATH = "../../clusters"
 SITE_PATH = "./site/docs"
 GALAXY_PATH = "../../galaxies"
 
-def save_cluster_relationships(cluster_data):
+def get_cluster_relationships(cluster_data):
     # Unpack cluster data
-    galaxy_name, cluster_name, cluster = cluster_data
+    galaxy, cluster = cluster_data
+    relationships = universe.get_relationships_with_levels(universe.galaxies[galaxy].clusters[cluster])
     
-    # Assuming `universe.get_relationships_with_levels` and `cluster.save_relationships`
+    print(f"Processed {galaxy}, {cluster}")
-    # are methods that can be called like this.
+    return cluster, galaxy, relationships
-    relationships = universe.get_relationships_with_levels(cluster)
-    cluster.save_relationships(relationships)
-    print(f"Processed {galaxy_name}, {cluster_name}")
 
 def get_deprecated_galaxy_files():
     deprecated_galaxy_files = []
@@ -46,24 +44,30 @@ def cluster_transform_to_link(cluster):
         .replace(":", "")
         .replace(placeholder, "-")
     )
-    return f"[{cluster.value} ({cluster.uuid})](../../{cluster.galaxy.json_file_name}/index.md#{section})"
+    galaxy_folder = cluster.galaxy.json_file_name.replace(".json", "")
+    return f"[{cluster.value} ({cluster.uuid})](../../{galaxy_folder}/index.md#{section})"
 
 def galaxy_transform_to_link(galaxy):
-    return f"[{galaxy.galaxy_name}](../../{galaxy.json_file_name}/index.md)"
+    galaxy_folder = galaxy.json_file_name.replace(".json", "")
+    return f"[{galaxy.galaxy_name}](../../{galaxy_folder}/index.md)"
 
 def generate_relations_table(relationships):
     markdown = "|Cluster A | Galaxy A | Cluster B | Galaxy B | Level { .graph } |\n"
-    markdown += "|--- | --- | --- | --- | --- | ---|\n"
+    markdown += "| --- | --- | --- | --- | --- |\n"
     for from_cluster, to_cluster, level in relationships:
-        from_galaxy = from_cluster.galaxy.galaxy_name
+        from_galaxy = from_cluster.galaxy
-        to_galaxy = to_cluster.galaxy.galaxy_name
+        if to_cluster.value != "Private Cluster":
-        markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {cluster_transform_to_link(to_cluster)} | {galaxy_transform_to_link(to_galaxy)} | {level}\n"
+            to_galaxy = to_cluster.galaxy
+            markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {cluster_transform_to_link(to_cluster)} | {galaxy_transform_to_link(to_galaxy)} | {level}\n"
+        else:
+            markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {to_cluster.value} | Unknown | {level}\n"
     return markdown
 
 def generate_index_page(galaxies):
     index_output = "# MISP Galaxy\n\nThe MISP galaxy offers a streamlined approach for representing large entities, known as clusters, which can be linked to MISP events or attributes. Each cluster consists of one or more elements, represented as key-value pairs. MISP galaxy comes with a default knowledge base, encompassing areas like Threat Actors, Tools, Ransomware, and ATT&CK matrices. However, users have the flexibility to modify, update, replace, or share these elements according to their needs.\n\nClusters and vocabularies within MISP galaxy can be utilized in their original form or as a foundational knowledge base. The distribution settings for each cluster can be adjusted, allowing for either restricted or wide dissemination.\n\nAdditionally, MISP galaxies enable the representation of existing standards like the MITRE ATT&CK™ framework, as well as custom matrices.\n\nThe aim is to provide a core set of clusters for organizations embarking on analysis, which can be further tailored to include localized, private information or additional, shareable data.\n\nClusters serve as an open and freely accessible knowledge base, which can be utilized and expanded within [MISP](https://www.misp-project.org/) or other threat intelligence platforms.\n\n![Overview of the integration of MISP galaxy in the MISP Threat Intelligence Sharing Platform](https://raw.githubusercontent.com/MISP/misp-galaxy/aa41337fd78946a60aef3783f58f337d2342430a/doc/images/galaxy.png)\n\n## Publicly available clusters\n"
     for galaxy in galaxies:
-        index_output += f"- [{galaxy.galaxy_name}](./{galaxy.json_file_name}/index.md)\n"
+        galaxy_folder = galaxy.json_file_name.replace(".json", "")
+        index_output += f"- [{galaxy.galaxy_name}](./{galaxy_folder}/index.md)\n"
     index_output += "## Statistics\n\nYou can find some statistics about MISP galaxies [here](./statistics.md).\n"
     index_output += "# Contributing\n\nIn the dynamic realm of threat intelligence, a variety of models and approaches exist to systematically organize, categorize, and delineate threat actors, hazards, or activity groups. We embrace innovative methodologies for articulating threat intelligence. The galaxy model is particularly versatile, enabling you to leverage and integrate methodologies that you trust and are already utilizing within your organization or community.\n\nWe encourage collaboration and contributions to the [MISP Galaxy JSON files](https://github.com/MISP/misp-galaxy/). Feel free to fork the project, enhance existing elements or clusters, or introduce new ones. Your insights are valuable - share them with us through a pull-request.\n"
     return index_output
@@ -93,7 +97,6 @@ if __name__ == "__main__":
                 meta=cluster.get("meta", None)
             )
 
-
     # Define the relationships between clusters
     for galaxy in galaxies_fnames:
         with open(os.path.join(CLUSTER_PATH, galaxy)) as fr:
@@ -102,20 +105,18 @@ if __name__ == "__main__":
                 if "related" in cluster:
                     for related in cluster["related"]:
                         universe.define_relationship(cluster["uuid"], related["dest-uuid"])
-    
-
-    # # Save relationships to clusters
-    # for galaxy in universe.galaxies.values():
-    #     for cluster in galaxy.clusters.values():
-    #         cluster.save_relationships(universe.get_relationships_with_levels(cluster))
 
     tasks = []
     for galaxy_name, galaxy in universe.galaxies.items():
         for cluster_name, cluster in galaxy.clusters.items():
-            tasks.append((galaxy_name, cluster_name, cluster))
+            tasks.append((galaxy_name, cluster_name))
 
     with Pool(processes=multiprocessing.cpu_count()) as pool:
-        pool.map(save_cluster_relationships, tasks)
+        result = pool.map(get_cluster_relationships, tasks)
+
+    for cluster, galaxy, relationships in result:
+        universe.galaxies[galaxy].clusters[cluster].relationships = relationships
+
     print("All clusters processed.")
 
     print(f"Finished relations in {time.time() - start_time} seconds")
@@ -131,9 +132,10 @@ if __name__ == "__main__":
         galaxy.write_entry(SITE_PATH)
 
     for galaxy in universe.galaxies.values():
-        if not os.path.exists(GALAXY_PATH):
+        galaxy_path = os.path.join(SITE_PATH, f"{galaxy.json_file_name}".replace(".json", ""))
-            os.mkdir(GALAXY_PATH)
+        if not os.path.exists(galaxy_path):
-        relation_path = os.path.join(GALAXY_PATH, "relations")
+            os.mkdir(galaxy_path)
+        relation_path = os.path.join(galaxy_path, "relations")
         if not os.path.exists(relation_path):
             os.mkdir(relation_path)
         with open(os.path.join(relation_path, ".pages"), "w") as index:
@@ -141,14 +143,8 @@ if __name__ == "__main__":
 
         for cluster in galaxy.clusters.values():
             if cluster.relationships:
+                print(f"Writing {cluster.uuid}.md")
                 with open(os.path.join(relation_path, f"{cluster.uuid}.md"), "w") as index:
                     index.write(generate_relations_table(cluster.relationships))
 
     print(f"Finished in {time.time() - start_time} seconds")
-
-    # relationships = universe.get_relationships_with_levels("Banker", "f0ec2df5-2e38-4df3-970d-525352006f2e")
-    # print(relationships)
-
-
-    # markdown_table = generate_markdown_table(relationships)
-    # print(markdown_table)

tools/mkdocs/modules/galaxy.py

@@ -16,9 +16,10 @@ class Galaxy:
             self.clusters[uuid] = Cluster(uuid=uuid, galaxy=self, description=description, value=value, meta=meta)
 
     def write_entry(self, path):
-        if not os.path.exists(path):
+        galaxy_path = os.path.join(path, f"{self.json_file_name}".replace(".json", ""))
-            os.mkdir(path)
+        if not os.path.exists(galaxy_path):
-        with open(os.path.join(path, f"{self.galaxy_name}.md"), "w") as index:
+            os.mkdir(galaxy_path)
+        with open(os.path.join(galaxy_path, "index.md"), "w") as index:
             index.write(self.generate_entry())
 
     def generate_entry(self):

tools/mkdocs/modules/universe.py

@@ -5,8 +5,9 @@ from collections import defaultdict, deque
 
 
 class Universe:
-    def __init__(self):
+    def __init__(self, add_inbound_relationship=False):
         self.galaxies = {}  # Maps galaxy_name to Galaxy objects
+        self.add_inbound_relationship = add_inbound_relationship
 
     def add_galaxy(self, galaxy_name, json_file_name, authors, description):
         if galaxy_name not in self.galaxies:
@@ -36,98 +37,10 @@ class Universe:
         else:
             # If Cluster B is not found, create a private cluster relationship for Cluster A
             if cluster_a:
-                private_cluster = Cluster(uuid=cluster_b_id, galaxy=None)
+                private_cluster = Cluster(uuid=cluster_b_id, galaxy=None, description=None, value="Private Cluster", meta=None)
                 cluster_a.add_outbound_relationship(private_cluster)
             else:
-                print("Cluster A not found in any galaxy")
+                raise ValueError(f"Cluster {cluster_a} not found in any galaxy")
-
-    # def get_relationships_with_levels(self, galaxy, cluster):
-    #     start_galaxy = self.galaxies[galaxy]
-    #     start_cluster = start_galaxy.clusters[cluster]
-
-    #     def bfs_with_inbound_outbound(start_cluster):
-    #         visited = set()  # To keep track of visited clusters
-    #         linked = set()  # To keep track of linked clusters
-    #         queue = deque([(start_cluster, 0, 'outbound')])  # Include direction of relationship
-    #         relationships = []
-
-    #         while queue:
-    #             current_cluster, level, direction = queue.popleft()
-    #             if (current_cluster, direction) not in visited:  # Check visited with direction
-    #                 visited.add((current_cluster, direction))
-
-    #                 # Process outbound relationships
-    #                 if direction == 'outbound':
-    #                     for to_cluster in current_cluster.outbound_relationships:
-    #                         if (to_cluster, 'outbound') not in visited:
-    #                             # relationships.append((current_cluster, to_cluster, level + 1, 'outbound'))
-    #                             queue.append((to_cluster, level + 1, 'outbound'))
-    #                             relationships.append((current_cluster, to_cluster, level + 1, 'outbound'))
-                            
-                    
-    #                 # Process inbound relationships
-    #                 for from_cluster in current_cluster.inbound_relationships:
-    #                     if (from_cluster, 'inbound') not in visited:
-    #                         relationships.append((from_cluster, current_cluster, level + 1, 'inbound'))
-    #                         queue.append((from_cluster, level + 1, 'inbound'))
-
-    #         return relationships
-
-
-    #     return bfs_with_inbound_outbound(start_cluster)
-
-    # def get_relationships_with_levels(self, galaxy, cluster):
-    #     start_galaxy = self.galaxies[galaxy]
-    #     start_cluster = start_galaxy.clusters[cluster]
-
-    #     def bfs_with_inbound_outbound(start_cluster):
-    #         visited = set()  # To keep track of visited clusters
-    #         relationships = defaultdict(lambda: (float('inf'), ''))  # Store lowest level for each link
-
-    #         queue = deque([(start_cluster, 0, 'outbound')])  # Include direction of relationship
-
-    #         while queue:
-    #             print(f"Queue: {[c.uuid for c, l, d in queue]}")
-    #             current_cluster, level, direction = queue.popleft()
-    #             if (current_cluster, direction) not in visited:  # Check visited with direction
-    #                 visited.add((current_cluster, direction))
-                    
-    #                 if current_cluster.uuid == "a5a067c9-c4d7-4f33-8e6f-01b903f89908":
-    #                     print(f"Current cluster: {current_cluster.uuid}, Level: {level}, Direction: {direction}")
-    #                     print(f"outbound relationships: {[x.uuid for x in current_cluster.outbound_relationships]}")
-
-
-    #                 # Process outbound relationships
-    #                 if direction == 'outbound':
-    #                     for to_cluster in current_cluster.outbound_relationships:
-    #                         if (to_cluster, 'outbound') not in visited:
-    #                             queue.append((to_cluster, level + 1, 'outbound'))
-
-    #                         link = frozenset([current_cluster, to_cluster])
-    #                         if relationships[link][0] > level + 1:
-    #                             relationships[link] = (level + 1, 'outbound')
-
-    #                 # Process inbound relationships
-    #                 for from_cluster in current_cluster.inbound_relationships:
-    #                     if (from_cluster, 'inbound') not in visited:
-    #                         queue.append((from_cluster, level + 1, 'inbound'))
-
-    #                     link = frozenset([from_cluster, current_cluster])
-    #                     if relationships[link][0] > level + 1:
-    #                         relationships[link] = (level + 1, 'inbound')
-
-    #         # Convert defaultdict to list of tuples for compatibility with your existing structure
-    #         processed_relationships = []
-    #         for link, (lvl, dir) in relationships.items():
-    #             clusters = list(link)
-    #             if dir == 'outbound':
-    #                 processed_relationships.append((clusters[0], clusters[1], lvl, dir))
-    #             else:
-    #                 processed_relationships.append((clusters[1], clusters[0], lvl, dir))
-
-    #         return processed_relationships
-
-    #     return bfs_with_inbound_outbound(start_cluster)
     
     def get_relationships_with_levels(self, start_cluster):
 
@@ -143,26 +56,37 @@ class Universe:
                     visited.add(current_cluster)
 
                     # Process all relationships regardless of direction
-                    neighbors = current_cluster.outbound_relationships.union(current_cluster.inbound_relationships)
+                    if self.add_inbound_relationship:
+                        neighbors = current_cluster.outbound_relationships.union(current_cluster.inbound_relationships)
+                    else:
+                        neighbors = current_cluster.outbound_relationships
                     for neighbor in neighbors:
                         link = frozenset([current_cluster, neighbor])
                         if level + 1 < relationships[link]:
                             relationships[link] = level + 1
-                            if neighbor not in visited:
+                            if neighbor not in visited and neighbor.value != "Private Cluster":
                                 queue.append((neighbor, level + 1))
-
+            
+            # count = 0
             # Convert the defaultdict to a list of tuples, ignoring direction
             processed_relationships = []
             for link, lvl in relationships.items():
                 # Extract clusters from the frozenset; direction is irrelevant
                 clusters = list(link)
+                if len(clusters) != 2:
+                    # count += 1
+                    continue
                 
                 # Arbitrarily choose the first cluster as 'source' for consistency
-                try: 
+                if clusters[0].value == "Private Cluster":
+                    processed_relationships.append((clusters[1], clusters[0], lvl))
+                else:
+                
                     processed_relationships.append((clusters[0], clusters[1], lvl))
-                except:
+                # except:
-                    processed_relationships.append((clusters[0], Cluster(uuid=0, galaxy=None), lvl))
+                #     processed_relationships.append((clusters[0], clusters[0], lvl)) # This is wrong just for testing!!!
 
+            # print(f"Count: {count}")
             return processed_relationships
 
         return bfs_with_undirected_relationships(start_cluster)