element-web/scripts/gen-workflow-mermaid.ts

640 lines
19 KiB
TypeScript
Raw Normal View History

#!/usr/bin/env -S npx ts-node
import fs from "node:fs";
import path from "node:path";
import YAML from "yaml";
import parseArgs from "minimist";
import cronstrue from "cronstrue";
import { partition } from "lodash";
const argv = parseArgs<{
debug: boolean;
on: string | string[];
}>(process.argv.slice(2), {
string: ["on"],
boolean: ["debug"],
});
/**
* Generates unique ID strings (incremental base36) representing the given inputs.
*/
class IdGenerator<T> {
private id = 0;
private map = new Map<T, string>();
public get(s: T): string {
if (this.map.has(s)) return this.map.get(s)!;
const id = "ID" + this.id.toString(36).toLowerCase();
this.map.set(s, id);
this.id++;
return id;
}
public debug(): void {
console.log("```");
console.log(this.map);
console.log("```");
}
}
/**
* Type representing a node on a graph with additional metadata
*/
interface Node {
// Workflows are keyed by project/name??id
// Jobs are keyed by id
// Triggers are keyed by id
id: string;
name: string;
shape:
| "round edges"
| "stadium"
| "subroutine"
| "cylinder"
| "circle"
| "flag"
| "rhombus"
| "hexagon"
| "parallelogram"
| "parallelogram_alt"
| "trapezoid"
| "trapezoid_alt"
| "double_circle";
link?: string;
}
/**
* Type representing a directed edge on a graph with an optional label
*/
type Edge<T> = [source: T, destination: T, label?: string];
class Graph<T extends Node> {
public nodes = new Map<string, T>();
public edges: Edge<T>[] = [];
public addNode(node: T): void {
if (!this.nodes.has(node.id)) {
this.nodes.set(node.id, node);
}
}
public removeNode(node: T): Edge<T>[] {
if (!this.nodes.has(node.id)) return [];
this.nodes.delete(node.id);
const [removedEdges, keptEdges] = partition(
this.edges,
([source, destination]) => source === node || destination === node,
);
this.edges = keptEdges;
return removedEdges;
}
public addEdge(source: T, destination: T, label?: string): void {
if (this.edges.some(([_source, _destination]) => _source === source && _destination === destination)) return;
this.edges.push([source, destination, label]);
}
// Removes nodes without any edges
public cull(): void {
const seenNodes = new Set<Node>();
graph.edges.forEach(([source, destination]) => {
seenNodes.add(source);
seenNodes.add(destination);
});
graph.nodes.forEach((node) => {
if (!seenNodes.has(node)) {
graph.nodes.delete(node.id);
}
});
}
public get roots(): Set<T> {
const roots = new Set(this.nodes.values());
this.edges.forEach(([source, destination]) => {
roots.delete(destination);
});
return roots;
}
private componentsRecurse(root: T, visited: Set<T>): T[] {
if (visited.has(root)) return [root];
visited.add(root);
const neighbours = [root];
this.edges.forEach(([source, destination]) => {
if (source === root) {
neighbours.push(...this.componentsRecurse(destination, visited));
} else if (destination === root) {
neighbours.push(...this.componentsRecurse(source, visited));
}
});
return neighbours;
}
public get components(): Graph<T>[] {
const graphs: Graph<T>[] = [];
const visited = new Set<T>();
this.nodes.forEach((node) => {
if (visited.has(node)) return;
const graph = new Graph<T>();
graphs.push(graph);
const nodes = this.componentsRecurse(node, visited);
nodes.forEach((node) => {
graph.addNode(node);
this.edges.forEach((edge) => {
if (edge[0] === node || edge[1] === node) {
graph.addEdge(...edge);
}
});
});
});
return graphs;
}
}
/**
* Type representing a GitHub project
*/
interface Project {
url: string;
name: string;
path: string;
workflows: Map<string, Workflow>;
}
/**
* Type representing a GitHub Actions Workflow
*/
interface Workflow extends Node {
path: string;
project: Project;
jobs: Job[];
on: WorkflowYaml["on"];
}
/**
* Type representing a job within a GitHub Actions Workflow
*/
interface Job extends Node {
jobId: string; // id relative to workflow
needs?: string[];
strategy?: {
matrix: {
[key: string]: string[];
} & {
include?: Record<string, string>[];
exclude?: Record<string, string>[];
};
};
}
/**
* Type representing the YAML structure of a GitHub Actions Workflow file
*/
interface WorkflowYaml {
name: string;
on: {
workflow_run?: {
workflows: string[];
}; // Magic
workflow_call?: {}; // Reusable
workflow_dispatch?: {}; // Manual
pull_request?: {};
merge_group?: {};
push?: {
tags?: string[];
branches?: string[];
};
schedule?: { cron: string }[];
release?: {};
//
label?: {};
issues?: {};
};
jobs: {
[job: string]: {
name?: string;
needs?: string | string[];
strategy?: Job["strategy"];
};
};
}
/**
* Type representing a trigger of a GitHub Actions Workflow
*/
type Trigger = Node;
// TODO workflow_call reusables
/* eslint-disable @typescript-eslint/naming-convention */
const TRIGGERS: {
[key in keyof WorkflowYaml["on"]]: (
data: NonNullable<WorkflowYaml["on"][key]>,
workflow: Workflow,
) => Trigger | Trigger[];
} = {
workflow_dispatch: () => ({
id: "on:workflow_dispatch",
name: "Manual",
shape: "circle",
}),
issues: (_, { project }) => ({ id: `on:issues/${project.name}`, name: `${project.name} Issues`, shape: "circle" }),
label: (_, { project }) => ({ id: "on:label", name: "on: Label", shape: "circle" }),
release: (_, { project }) => ({
id: `on:release/${project.name}`,
name: `${project.name} Release`,
shape: "circle",
}),
push: (data, { project }) => {
const nodes: Trigger[] = [];
data.tags?.forEach((tag) => {
const name = `Push ${project.name}<br>tag ${tag}`;
nodes.push({ id: `on:push/${project.name}/tag/${tag}`, name, shape: "circle" });
});
data.branches?.forEach((branch) => {
const name = `Push ${project.name}<br>${branch}`;
nodes.push({ id: `on:push/${project.name}/branch/${branch}`, name, shape: "circle" });
});
return nodes;
},
schedule: (data) =>
data.map(({ cron }) => ({
id: `on:schedule/${cron}`,
name: cronstrue.toString(cron).replaceAll(", ", "<br>"),
shape: "circle",
})),
pull_request: (_, { project }) => ({
id: `on:pull_request/${project.name}`,
name: `Pull Request<br>${project.name}`,
shape: "circle",
}),
// TODO should we be just dropping these?
workflow_run: (data) => data.workflows.map((parent) => workflows.get(parent)).filter(Boolean) as Workflow[],
};
/* eslint-enable @typescript-eslint/naming-convention */
const triggers = new Map<string, Trigger>(); // keyed by trigger id
const projects = new Map<string, Project>(); // keyed by project name
const workflows = new Map<string, Workflow>(); // keyed by workflow name
function getTriggerNodes<K extends keyof WorkflowYaml["on"]>(key: K, workflow: Workflow): Trigger[] {
if (!TRIGGERS[key]) return [];
if ((typeof argv.on === "string" || Array.isArray(argv.on)) && !toArray(argv.on).includes(key)) {
return [];
}
const data = workflow.on[key]!;
const nodes = toArray(TRIGGERS[key]!(data, workflow));
return nodes.map((node) => {
if (triggers.has(node.id)) return triggers.get(node.id)!;
triggers.set(node.id, node);
return node;
});
}
function readFile(...pathSegments: string[]): string {
return fs.readFileSync(path.join(...pathSegments), { encoding: "utf-8" });
}
function readJson<T extends object>(...pathSegments: string[]): T {
return JSON.parse(readFile(...pathSegments));
}
function readYaml<T extends object>(...pathSegments: string[]): T {
return YAML.parse(readFile(...pathSegments));
}
function toArray<T>(v: T | T[]): T[] {
return Array.isArray(v) ? v : [v];
}
function cartesianProduct<T>(sets: T[][]): T[][] {
return sets.reduce<T[][]>(
(results, ids) =>
results
.map((result) => ids.map((id) => [...result, id]))
.reduce((nested, result) => [...nested, ...result]),
[[]],
);
}
function shallowCompare(obj1: Record<string, any>, obj2: Record<string, any>): boolean {
return (
Object.keys(obj1).length === Object.keys(obj2).length &&
Object.keys(obj1).every((key) => obj1[key] === obj2[key])
);
}
// Data ingest
for (const projectPath of argv._) {
const {
name,
repository: { url },
} = readJson<{ name: string; repository: { url: string } }>(projectPath, "package.json");
const workflowsPath = path.join(projectPath, ".github", "workflows");
const project: Project = {
name,
url,
path: projectPath,
workflows: new Map(),
};
for (const file of fs.readdirSync(workflowsPath).filter((f) => f.endsWith(".yml") || f.endsWith(".yaml"))) {
const data = readYaml<WorkflowYaml>(workflowsPath, file);
const name = data.name ?? file;
const workflow: Workflow = {
id: `${project.name}/${name}`,
name,
shape: "hexagon",
path: path.join(workflowsPath, file),
project,
link: `${project.url}/blob/develop/.github/workflows/${file}`,
on: data.on,
jobs: [],
};
for (const jobId in data.jobs) {
const job = data.jobs[jobId];
workflow.jobs.push({
id: `${workflow.name}/${jobId}`,
jobId,
name: job.name ?? jobId,
strategy: job.strategy,
needs: job.needs ? toArray(job.needs) : undefined,
shape: "subroutine",
link: `${project.url}/blob/develop/.github/workflows/${file}`,
});
}
project.workflows.set(name, workflow);
workflows.set(name, workflow);
}
projects.set(name, project);
}
class MermaidFlowchartPrinter {
private static INDENT = 4;
private currentIndent = 0;
private text = "";
public readonly idGenerator = new IdGenerator();
private print(text: string): void {
this.text += " ".repeat(this.currentIndent) + text + "\n";
}
public finish(): void {
this.indent(-1);
if (this.markdown) this.print("```\n");
console.log(this.text);
}
private indent(delta = 1): void {
this.currentIndent += delta * MermaidFlowchartPrinter.INDENT;
}
public constructor(
direction: "TD" | "TB" | "BT" | "RL" | "LR",
title?: string,
private readonly markdown = false,
) {
if (this.markdown) {
this.print("```mermaid");
}
// Print heading
if (title) {
this.print("---");
this.print(`title: ${title}`);
this.print("---");
}
this.print(`flowchart ${direction}`);
this.indent();
}
public subgraph(id: string, name: string, fn: () => void): void {
this.print(`subgraph ${this.idGenerator.get(id)}["${name}"]`);
this.indent();
fn();
this.indent(-1);
this.print("end");
}
public node(node: Node): void {
const id = this.idGenerator.get(node.id);
const name = node.name.replaceAll('"', "'");
switch (node.shape) {
case "round edges":
this.print(`${id}("${name}")`);
break;
case "stadium":
this.print(`${id}(["${name}"])`);
break;
case "subroutine":
this.print(`${id}[["${name}"]]`);
break;
case "cylinder":
this.print(`${id}[("${name}")]`);
break;
case "circle":
this.print(`${id}(("${name}"))`);
break;
case "flag":
this.print(`${id}>"${name}"]`);
break;
case "rhombus":
this.print(`${id}{"${name}"}`);
break;
case "hexagon":
this.print(`${id}{{"${name}"}}`);
break;
case "parallelogram":
this.print(`${id}[/"${name}"/]`);
break;
case "parallelogram_alt":
this.print(`${id}[\\"${name}"\\]`);
break;
case "trapezoid":
this.print(`${id}[/"${name}"\\]`);
break;
case "trapezoid_alt":
this.print(`${id}[\\"${name}"/]`);
break;
case "double_circle":
this.print(`${id}((("${name}")))`);
break;
}
if (node.link) {
this.print(`click ${id} href "${node.link}" "Click to open workflow"`);
}
}
public edge(source: Node, destination: Node, text?: string): void {
const sourceId = this.idGenerator.get(source.id);
const destinationId = this.idGenerator.get(destination.id);
if (text) {
this.print(`${sourceId}-- ${text} -->${destinationId}`);
} else {
this.print(`${sourceId} --> ${destinationId}`);
}
}
}
const graph = new Graph<Workflow | Node>();
for (const workflow of workflows.values()) {
if (
(typeof argv.on === "string" || Array.isArray(argv.on)) &&
!toArray(argv.on).some((trigger) => trigger in workflow.on)
) {
continue;
}
graph.addNode(workflow);
Object.keys(workflow.on).forEach((trigger) => {
const nodes = getTriggerNodes(trigger as keyof WorkflowYaml["on"], workflow);
nodes.forEach((node) => {
graph.addNode(node);
graph.addEdge(node, workflow, "project" in node ? "workflow_run" : undefined);
});
});
}
// TODO separate disconnected nodes into their own graph
graph.cull();
// This is an awful hack to make the output graphs much better by allowing the splitting of certain nodes //
const bifurcatedNodes = [triggers.get("on:workflow_dispatch")].filter(Boolean) as Node[];
const removedEdgeMap = new Map<Node, Edge<any>[]>();
for (const node of bifurcatedNodes) {
removedEdgeMap.set(node, graph.removeNode(node));
}
const components = graph.components;
for (const node of bifurcatedNodes) {
const removedEdges = removedEdgeMap.get(node)!;
components.forEach((graph) => {
removedEdges.forEach((edge) => {
if (graph.nodes.has(edge[1].id)) {
graph.addNode(node);
graph.addEdge(...edge);
}
});
});
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////
if (argv.debug) {
debugGraph("global", graph);
}
components.forEach((graph) => {
const title = [...graph.roots]
.map((root) => root.name)
.join(" & ")
.replaceAll("<br>", " ");
const printer = new MermaidFlowchartPrinter("LR", title, true);
graph.nodes.forEach((node) => {
if ("project" in node) {
// TODO unsure about this edge
// if (node.jobs.length === 1) {
// printer.node(node);
// return;
// }
// TODO handle job.if on github.event_name
const subgraph = new Graph<Job>();
for (const job of node.jobs) {
subgraph.addNode(job);
if (job.needs) {
toArray(job.needs).forEach((req) => {
subgraph.addEdge(node.jobs.find((job) => job.jobId === req)!, job, "needs");
});
}
}
printer.subgraph(node.id, node.name, () => {
subgraph.edges.forEach(([source, destination, text]) => {
printer.edge(source, destination, text);
});
subgraph.nodes.forEach((job) => {
if (!job.strategy?.matrix) {
printer.node(job);
return;
}
let variations = cartesianProduct(
Object.keys(job.strategy.matrix)
.filter((key) => key !== "include" && key !== "exclude")
.map((matrixKey) => {
return job.strategy!.matrix[matrixKey].map((value) => ({ [matrixKey]: value }));
}),
)
.map((variation) => Object.assign({}, ...variation))
.filter((variation) => Object.keys(variation).length > 0);
if (job.strategy.matrix.include) {
variations.push(...job.strategy.matrix.include);
}
job.strategy.matrix.exclude?.forEach((exclusion) => {
variations = variations.filter((variation) => {
return !shallowCompare(exclusion, variation);
});
});
// TODO validate edge case
if (variations.length === 0) {
printer.node(job);
return;
}
const jobName = job.name.replace(/\${{.+}}/g, "").replace(/(?:\(\)| )+/g, " ");
printer.subgraph(job.id, jobName, () => {
variations.forEach((variation, i) => {
let variationName = job.name;
if (variationName.includes("${{ matrix.")) {
Object.keys(variation).map((key) => {
variationName = variationName.replace(`\${{ matrix.${key} }}`, variation[key]);
});
} else {
variationName = `${variationName} (${Object.values(variation).join(", ")})`;
}
printer.node({ ...job, id: `${job.id}-variation-${i}`, name: variationName });
});
});
});
});
return;
}
printer.node(node);
});
graph.edges.forEach(([sourceName, destinationName, text]) => {
printer.edge(sourceName, destinationName, text);
});
printer.finish();
if (argv.debug) {
printer.idGenerator.debug();
debugGraph("subgraph", graph);
}
});
function debugGraph(name: string, graph: Graph<any>): void {
console.log("```");
console.log(`## ${name}`);
console.log(new Map(graph.nodes));
console.log(graph.edges.map((edge) => ({ source: edge[0].id, destination: edge[1].id, text: edge[2] })));
console.log("```");
console.log("");
}