640 lines
19 KiB
TypeScript
Executable File
640 lines
19 KiB
TypeScript
Executable File
#!/usr/bin/env -S npx ts-node
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import YAML from "yaml";
|
|
import parseArgs from "minimist";
|
|
import cronstrue from "cronstrue";
|
|
import { partition } from "lodash";
|
|
|
|
const argv = parseArgs<{
|
|
debug: boolean;
|
|
on: string | string[];
|
|
}>(process.argv.slice(2), {
|
|
string: ["on"],
|
|
boolean: ["debug"],
|
|
});
|
|
|
|
/**
|
|
* Generates unique ID strings (incremental base36) representing the given inputs.
|
|
*/
|
|
class IdGenerator<T> {
|
|
private id = 0;
|
|
private map = new Map<T, string>();
|
|
|
|
public get(s: T): string {
|
|
if (this.map.has(s)) return this.map.get(s)!;
|
|
const id = "ID" + this.id.toString(36).toLowerCase();
|
|
this.map.set(s, id);
|
|
this.id++;
|
|
return id;
|
|
}
|
|
|
|
public debug(): void {
|
|
console.log("```");
|
|
console.log(this.map);
|
|
console.log("```");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Type representing a node on a graph with additional metadata
|
|
*/
|
|
interface Node {
|
|
// Workflows are keyed by project/name??id
|
|
// Jobs are keyed by id
|
|
// Triggers are keyed by id
|
|
id: string;
|
|
name: string;
|
|
shape:
|
|
| "round edges"
|
|
| "stadium"
|
|
| "subroutine"
|
|
| "cylinder"
|
|
| "circle"
|
|
| "flag"
|
|
| "rhombus"
|
|
| "hexagon"
|
|
| "parallelogram"
|
|
| "parallelogram_alt"
|
|
| "trapezoid"
|
|
| "trapezoid_alt"
|
|
| "double_circle";
|
|
link?: string;
|
|
}
|
|
|
|
/**
|
|
* Type representing a directed edge on a graph with an optional label
|
|
*/
|
|
type Edge<T> = [source: T, destination: T, label?: string];
|
|
|
|
class Graph<T extends Node> {
|
|
public nodes = new Map<string, T>();
|
|
public edges: Edge<T>[] = [];
|
|
|
|
public addNode(node: T): void {
|
|
if (!this.nodes.has(node.id)) {
|
|
this.nodes.set(node.id, node);
|
|
}
|
|
}
|
|
|
|
public removeNode(node: T): Edge<T>[] {
|
|
if (!this.nodes.has(node.id)) return [];
|
|
this.nodes.delete(node.id);
|
|
const [removedEdges, keptEdges] = partition(
|
|
this.edges,
|
|
([source, destination]) => source === node || destination === node,
|
|
);
|
|
this.edges = keptEdges;
|
|
return removedEdges;
|
|
}
|
|
|
|
public addEdge(source: T, destination: T, label?: string): void {
|
|
if (this.edges.some(([_source, _destination]) => _source === source && _destination === destination)) return;
|
|
this.edges.push([source, destination, label]);
|
|
}
|
|
|
|
// Removes nodes without any edges
|
|
public cull(): void {
|
|
const seenNodes = new Set<Node>();
|
|
graph.edges.forEach(([source, destination]) => {
|
|
seenNodes.add(source);
|
|
seenNodes.add(destination);
|
|
});
|
|
graph.nodes.forEach((node) => {
|
|
if (!seenNodes.has(node)) {
|
|
graph.nodes.delete(node.id);
|
|
}
|
|
});
|
|
}
|
|
|
|
public get roots(): Set<T> {
|
|
const roots = new Set(this.nodes.values());
|
|
this.edges.forEach(([source, destination]) => {
|
|
roots.delete(destination);
|
|
});
|
|
return roots;
|
|
}
|
|
|
|
private componentsRecurse(root: T, visited: Set<T>): T[] {
|
|
if (visited.has(root)) return [root];
|
|
visited.add(root);
|
|
|
|
const neighbours = [root];
|
|
this.edges.forEach(([source, destination]) => {
|
|
if (source === root) {
|
|
neighbours.push(...this.componentsRecurse(destination, visited));
|
|
} else if (destination === root) {
|
|
neighbours.push(...this.componentsRecurse(source, visited));
|
|
}
|
|
});
|
|
|
|
return neighbours;
|
|
}
|
|
|
|
public get components(): Graph<T>[] {
|
|
const graphs: Graph<T>[] = [];
|
|
const visited = new Set<T>();
|
|
this.nodes.forEach((node) => {
|
|
if (visited.has(node)) return;
|
|
|
|
const graph = new Graph<T>();
|
|
graphs.push(graph);
|
|
|
|
const nodes = this.componentsRecurse(node, visited);
|
|
nodes.forEach((node) => {
|
|
graph.addNode(node);
|
|
this.edges.forEach((edge) => {
|
|
if (edge[0] === node || edge[1] === node) {
|
|
graph.addEdge(...edge);
|
|
}
|
|
});
|
|
});
|
|
});
|
|
|
|
return graphs;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Type representing a GitHub project
|
|
*/
|
|
interface Project {
|
|
url: string;
|
|
name: string;
|
|
path: string;
|
|
workflows: Map<string, Workflow>;
|
|
}
|
|
|
|
/**
|
|
* Type representing a GitHub Actions Workflow
|
|
*/
|
|
interface Workflow extends Node {
|
|
path: string;
|
|
project: Project;
|
|
jobs: Job[];
|
|
on: WorkflowYaml["on"];
|
|
}
|
|
|
|
/**
|
|
* Type representing a job within a GitHub Actions Workflow
|
|
*/
|
|
interface Job extends Node {
|
|
jobId: string; // id relative to workflow
|
|
needs?: string[];
|
|
strategy?: {
|
|
matrix: {
|
|
[key: string]: string[];
|
|
} & {
|
|
include?: Record<string, string>[];
|
|
exclude?: Record<string, string>[];
|
|
};
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Type representing the YAML structure of a GitHub Actions Workflow file
|
|
*/
|
|
interface WorkflowYaml {
|
|
name: string;
|
|
on: {
|
|
workflow_run?: {
|
|
workflows: string[];
|
|
}; // Magic
|
|
workflow_call?: {}; // Reusable
|
|
workflow_dispatch?: {}; // Manual
|
|
pull_request?: {};
|
|
merge_group?: {};
|
|
push?: {
|
|
tags?: string[];
|
|
branches?: string[];
|
|
};
|
|
schedule?: { cron: string }[];
|
|
release?: {};
|
|
//
|
|
label?: {};
|
|
issues?: {};
|
|
};
|
|
jobs: {
|
|
[job: string]: {
|
|
name?: string;
|
|
needs?: string | string[];
|
|
strategy?: Job["strategy"];
|
|
};
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Type representing a trigger of a GitHub Actions Workflow
|
|
*/
|
|
type Trigger = Node;
|
|
|
|
// TODO workflow_call reusables
|
|
/* eslint-disable @typescript-eslint/naming-convention */
|
|
const TRIGGERS: {
|
|
[key in keyof WorkflowYaml["on"]]: (
|
|
data: NonNullable<WorkflowYaml["on"][key]>,
|
|
workflow: Workflow,
|
|
) => Trigger | Trigger[];
|
|
} = {
|
|
workflow_dispatch: () => ({
|
|
id: "on:workflow_dispatch",
|
|
name: "Manual",
|
|
shape: "circle",
|
|
}),
|
|
issues: (_, { project }) => ({ id: `on:issues/${project.name}`, name: `${project.name} Issues`, shape: "circle" }),
|
|
label: (_, { project }) => ({ id: "on:label", name: "on: Label", shape: "circle" }),
|
|
release: (_, { project }) => ({
|
|
id: `on:release/${project.name}`,
|
|
name: `${project.name} Release`,
|
|
shape: "circle",
|
|
}),
|
|
push: (data, { project }) => {
|
|
const nodes: Trigger[] = [];
|
|
data.tags?.forEach((tag) => {
|
|
const name = `Push ${project.name}<br>tag ${tag}`;
|
|
nodes.push({ id: `on:push/${project.name}/tag/${tag}`, name, shape: "circle" });
|
|
});
|
|
data.branches?.forEach((branch) => {
|
|
const name = `Push ${project.name}<br>${branch}`;
|
|
nodes.push({ id: `on:push/${project.name}/branch/${branch}`, name, shape: "circle" });
|
|
});
|
|
return nodes;
|
|
},
|
|
schedule: (data) =>
|
|
data.map(({ cron }) => ({
|
|
id: `on:schedule/${cron}`,
|
|
name: cronstrue.toString(cron).replaceAll(", ", "<br>"),
|
|
shape: "circle",
|
|
})),
|
|
pull_request: (_, { project }) => ({
|
|
id: `on:pull_request/${project.name}`,
|
|
name: `Pull Request<br>${project.name}`,
|
|
shape: "circle",
|
|
}),
|
|
// TODO should we be just dropping these?
|
|
workflow_run: (data) => data.workflows.map((parent) => workflows.get(parent)).filter(Boolean) as Workflow[],
|
|
};
|
|
/* eslint-enable @typescript-eslint/naming-convention */
|
|
|
|
const triggers = new Map<string, Trigger>(); // keyed by trigger id
|
|
const projects = new Map<string, Project>(); // keyed by project name
|
|
const workflows = new Map<string, Workflow>(); // keyed by workflow name
|
|
|
|
function getTriggerNodes<K extends keyof WorkflowYaml["on"]>(key: K, workflow: Workflow): Trigger[] {
|
|
if (!TRIGGERS[key]) return [];
|
|
|
|
if ((typeof argv.on === "string" || Array.isArray(argv.on)) && !toArray(argv.on).includes(key)) {
|
|
return [];
|
|
}
|
|
|
|
const data = workflow.on[key]!;
|
|
const nodes = toArray(TRIGGERS[key]!(data, workflow));
|
|
return nodes.map((node) => {
|
|
if (triggers.has(node.id)) return triggers.get(node.id)!;
|
|
triggers.set(node.id, node);
|
|
return node;
|
|
});
|
|
}
|
|
|
|
function readFile(...pathSegments: string[]): string {
|
|
return fs.readFileSync(path.join(...pathSegments), { encoding: "utf-8" });
|
|
}
|
|
|
|
function readJson<T extends object>(...pathSegments: string[]): T {
|
|
return JSON.parse(readFile(...pathSegments));
|
|
}
|
|
|
|
function readYaml<T extends object>(...pathSegments: string[]): T {
|
|
return YAML.parse(readFile(...pathSegments));
|
|
}
|
|
|
|
function toArray<T>(v: T | T[]): T[] {
|
|
return Array.isArray(v) ? v : [v];
|
|
}
|
|
|
|
function cartesianProduct<T>(sets: T[][]): T[][] {
|
|
return sets.reduce<T[][]>(
|
|
(results, ids) =>
|
|
results
|
|
.map((result) => ids.map((id) => [...result, id]))
|
|
.reduce((nested, result) => [...nested, ...result]),
|
|
[[]],
|
|
);
|
|
}
|
|
|
|
function shallowCompare(obj1: Record<string, any>, obj2: Record<string, any>): boolean {
|
|
return (
|
|
Object.keys(obj1).length === Object.keys(obj2).length &&
|
|
Object.keys(obj1).every((key) => obj1[key] === obj2[key])
|
|
);
|
|
}
|
|
|
|
// Data ingest
|
|
for (const projectPath of argv._) {
|
|
const {
|
|
name,
|
|
repository: { url },
|
|
} = readJson<{ name: string; repository: { url: string } }>(projectPath, "package.json");
|
|
const workflowsPath = path.join(projectPath, ".github", "workflows");
|
|
|
|
const project: Project = {
|
|
name,
|
|
url,
|
|
path: projectPath,
|
|
workflows: new Map(),
|
|
};
|
|
|
|
for (const file of fs.readdirSync(workflowsPath).filter((f) => f.endsWith(".yml") || f.endsWith(".yaml"))) {
|
|
const data = readYaml<WorkflowYaml>(workflowsPath, file);
|
|
const name = data.name ?? file;
|
|
const workflow: Workflow = {
|
|
id: `${project.name}/${name}`,
|
|
name,
|
|
shape: "hexagon",
|
|
path: path.join(workflowsPath, file),
|
|
project,
|
|
link: `${project.url}/blob/develop/.github/workflows/${file}`,
|
|
|
|
on: data.on,
|
|
jobs: [],
|
|
};
|
|
|
|
for (const jobId in data.jobs) {
|
|
const job = data.jobs[jobId];
|
|
workflow.jobs.push({
|
|
id: `${workflow.name}/${jobId}`,
|
|
jobId,
|
|
name: job.name ?? jobId,
|
|
strategy: job.strategy,
|
|
needs: job.needs ? toArray(job.needs) : undefined,
|
|
shape: "subroutine",
|
|
link: `${project.url}/blob/develop/.github/workflows/${file}`,
|
|
});
|
|
}
|
|
|
|
project.workflows.set(name, workflow);
|
|
workflows.set(name, workflow);
|
|
}
|
|
|
|
projects.set(name, project);
|
|
}
|
|
|
|
class MermaidFlowchartPrinter {
|
|
private static INDENT = 4;
|
|
private currentIndent = 0;
|
|
private text = "";
|
|
public readonly idGenerator = new IdGenerator();
|
|
|
|
private print(text: string): void {
|
|
this.text += " ".repeat(this.currentIndent) + text + "\n";
|
|
}
|
|
|
|
public finish(): void {
|
|
this.indent(-1);
|
|
if (this.markdown) this.print("```\n");
|
|
console.log(this.text);
|
|
}
|
|
|
|
private indent(delta = 1): void {
|
|
this.currentIndent += delta * MermaidFlowchartPrinter.INDENT;
|
|
}
|
|
|
|
public constructor(
|
|
direction: "TD" | "TB" | "BT" | "RL" | "LR",
|
|
title?: string,
|
|
private readonly markdown = false,
|
|
) {
|
|
if (this.markdown) {
|
|
this.print("```mermaid");
|
|
}
|
|
// Print heading
|
|
if (title) {
|
|
this.print("---");
|
|
this.print(`title: ${title}`);
|
|
this.print("---");
|
|
}
|
|
this.print(`flowchart ${direction}`);
|
|
this.indent();
|
|
}
|
|
|
|
public subgraph(id: string, name: string, fn: () => void): void {
|
|
this.print(`subgraph ${this.idGenerator.get(id)}["${name}"]`);
|
|
this.indent();
|
|
fn();
|
|
this.indent(-1);
|
|
this.print("end");
|
|
}
|
|
|
|
public node(node: Node): void {
|
|
const id = this.idGenerator.get(node.id);
|
|
const name = node.name.replaceAll('"', "'");
|
|
switch (node.shape) {
|
|
case "round edges":
|
|
this.print(`${id}("${name}")`);
|
|
break;
|
|
case "stadium":
|
|
this.print(`${id}(["${name}"])`);
|
|
break;
|
|
case "subroutine":
|
|
this.print(`${id}[["${name}"]]`);
|
|
break;
|
|
case "cylinder":
|
|
this.print(`${id}[("${name}")]`);
|
|
break;
|
|
case "circle":
|
|
this.print(`${id}(("${name}"))`);
|
|
break;
|
|
case "flag":
|
|
this.print(`${id}>"${name}"]`);
|
|
break;
|
|
case "rhombus":
|
|
this.print(`${id}{"${name}"}`);
|
|
break;
|
|
case "hexagon":
|
|
this.print(`${id}{{"${name}"}}`);
|
|
break;
|
|
case "parallelogram":
|
|
this.print(`${id}[/"${name}"/]`);
|
|
break;
|
|
case "parallelogram_alt":
|
|
this.print(`${id}[\\"${name}"\\]`);
|
|
break;
|
|
case "trapezoid":
|
|
this.print(`${id}[/"${name}"\\]`);
|
|
break;
|
|
case "trapezoid_alt":
|
|
this.print(`${id}[\\"${name}"/]`);
|
|
break;
|
|
case "double_circle":
|
|
this.print(`${id}((("${name}")))`);
|
|
break;
|
|
}
|
|
|
|
if (node.link) {
|
|
this.print(`click ${id} href "${node.link}" "Click to open workflow"`);
|
|
}
|
|
}
|
|
|
|
public edge(source: Node, destination: Node, text?: string): void {
|
|
const sourceId = this.idGenerator.get(source.id);
|
|
const destinationId = this.idGenerator.get(destination.id);
|
|
if (text) {
|
|
this.print(`${sourceId}-- ${text} -->${destinationId}`);
|
|
} else {
|
|
this.print(`${sourceId} --> ${destinationId}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const graph = new Graph<Workflow | Node>();
|
|
for (const workflow of workflows.values()) {
|
|
if (
|
|
(typeof argv.on === "string" || Array.isArray(argv.on)) &&
|
|
!toArray(argv.on).some((trigger) => trigger in workflow.on)
|
|
) {
|
|
continue;
|
|
}
|
|
|
|
graph.addNode(workflow);
|
|
Object.keys(workflow.on).forEach((trigger) => {
|
|
const nodes = getTriggerNodes(trigger as keyof WorkflowYaml["on"], workflow);
|
|
nodes.forEach((node) => {
|
|
graph.addNode(node);
|
|
graph.addEdge(node, workflow, "project" in node ? "workflow_run" : undefined);
|
|
});
|
|
});
|
|
}
|
|
|
|
// TODO separate disconnected nodes into their own graph
|
|
graph.cull();
|
|
|
|
// This is an awful hack to make the output graphs much better by allowing the splitting of certain nodes //
|
|
const bifurcatedNodes = [triggers.get("on:workflow_dispatch")].filter(Boolean) as Node[];
|
|
const removedEdgeMap = new Map<Node, Edge<any>[]>();
|
|
for (const node of bifurcatedNodes) {
|
|
removedEdgeMap.set(node, graph.removeNode(node));
|
|
}
|
|
|
|
const components = graph.components;
|
|
for (const node of bifurcatedNodes) {
|
|
const removedEdges = removedEdgeMap.get(node)!;
|
|
components.forEach((graph) => {
|
|
removedEdges.forEach((edge) => {
|
|
if (graph.nodes.has(edge[1].id)) {
|
|
graph.addNode(node);
|
|
graph.addEdge(...edge);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
if (argv.debug) {
|
|
debugGraph("global", graph);
|
|
}
|
|
|
|
components.forEach((graph) => {
|
|
const title = [...graph.roots]
|
|
.map((root) => root.name)
|
|
.join(" & ")
|
|
.replaceAll("<br>", " ");
|
|
const printer = new MermaidFlowchartPrinter("LR", title, true);
|
|
graph.nodes.forEach((node) => {
|
|
if ("project" in node) {
|
|
// TODO unsure about this edge
|
|
// if (node.jobs.length === 1) {
|
|
// printer.node(node);
|
|
// return;
|
|
// }
|
|
|
|
// TODO handle job.if on github.event_name
|
|
|
|
const subgraph = new Graph<Job>();
|
|
for (const job of node.jobs) {
|
|
subgraph.addNode(job);
|
|
if (job.needs) {
|
|
toArray(job.needs).forEach((req) => {
|
|
subgraph.addEdge(node.jobs.find((job) => job.jobId === req)!, job, "needs");
|
|
});
|
|
}
|
|
}
|
|
|
|
printer.subgraph(node.id, node.name, () => {
|
|
subgraph.edges.forEach(([source, destination, text]) => {
|
|
printer.edge(source, destination, text);
|
|
});
|
|
|
|
subgraph.nodes.forEach((job) => {
|
|
if (!job.strategy?.matrix) {
|
|
printer.node(job);
|
|
return;
|
|
}
|
|
|
|
let variations = cartesianProduct(
|
|
Object.keys(job.strategy.matrix)
|
|
.filter((key) => key !== "include" && key !== "exclude")
|
|
.map((matrixKey) => {
|
|
return job.strategy!.matrix[matrixKey].map((value) => ({ [matrixKey]: value }));
|
|
}),
|
|
)
|
|
.map((variation) => Object.assign({}, ...variation))
|
|
.filter((variation) => Object.keys(variation).length > 0);
|
|
|
|
if (job.strategy.matrix.include) {
|
|
variations.push(...job.strategy.matrix.include);
|
|
}
|
|
job.strategy.matrix.exclude?.forEach((exclusion) => {
|
|
variations = variations.filter((variation) => {
|
|
return !shallowCompare(exclusion, variation);
|
|
});
|
|
});
|
|
|
|
// TODO validate edge case
|
|
if (variations.length === 0) {
|
|
printer.node(job);
|
|
return;
|
|
}
|
|
|
|
const jobName = job.name.replace(/\${{.+}}/g, "").replace(/(?:\(\)| )+/g, " ");
|
|
printer.subgraph(job.id, jobName, () => {
|
|
variations.forEach((variation, i) => {
|
|
let variationName = job.name;
|
|
if (variationName.includes("${{ matrix.")) {
|
|
Object.keys(variation).map((key) => {
|
|
variationName = variationName.replace(`\${{ matrix.${key} }}`, variation[key]);
|
|
});
|
|
} else {
|
|
variationName = `${variationName} (${Object.values(variation).join(", ")})`;
|
|
}
|
|
|
|
printer.node({ ...job, id: `${job.id}-variation-${i}`, name: variationName });
|
|
});
|
|
});
|
|
});
|
|
});
|
|
return;
|
|
}
|
|
|
|
printer.node(node);
|
|
});
|
|
graph.edges.forEach(([sourceName, destinationName, text]) => {
|
|
printer.edge(sourceName, destinationName, text);
|
|
});
|
|
printer.finish();
|
|
|
|
if (argv.debug) {
|
|
printer.idGenerator.debug();
|
|
debugGraph("subgraph", graph);
|
|
}
|
|
});
|
|
|
|
function debugGraph(name: string, graph: Graph<any>): void {
|
|
console.log("```");
|
|
console.log(`## ${name}`);
|
|
console.log(new Map(graph.nodes));
|
|
console.log(graph.edges.map((edge) => ({ source: edge[0].id, destination: edge[1].id, text: edge[2] })));
|
|
console.log("```");
|
|
console.log("");
|
|
}
|