Merge pull request #26 from checkly/feature/github-release-aggregator

Update GitHub context aggregator to include release summaries

prisma/migrations/20241213143026_add_on_delete_cascade/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "AlertContext" DROP CONSTRAINT "AlertContext_alertId_fkey";
+
+-- AddForeignKey
+ALTER TABLE "AlertContext" ADD CONSTRAINT "AlertContext_alertId_fkey" FOREIGN KEY ("alertId") REFERENCES "Alert"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -52,5 +52,5 @@ model AlertContext {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
-  alert Alert @relation(fields: [alertId], references: [id])
+  alert Alert @relation(fields: [alertId], references: [id], onDelete: Cascade)
 }

src/aggregator/ContextAggregator.ts

@@ -1,6 +1,7 @@
 import { checklyAggregator } from "./checkly-aggregator";
 import { WebhookAlertDto } from "../checkly/alertDTO";
 import { githubAggregator } from "./github-aggregator";
+import type { $Enums } from "@prisma/client";
 
 export enum ContextKey {
 	ChecklyScript = "checkly.script",
@@ -10,11 +11,12 @@ export enum ContextKey {
 	ChecklyPrometheusStatus = "checkly.prometheusStatus",
 	ChecklyLogs = "checkly.logs",
 	GitHubRepoChanges = "github.repoChanges.$repo",
+	GitHubReleaseSummary = "github.releaseSummary.$repo",
 }
 
 export interface CheckContext {
 	checkId: string;
-	source: "checkly" | "github";
+	source: $Enums.Source;
 	key: ContextKey;
 	value: unknown;
 	analysis: string;

src/aggregator/chains.ts

@@ -5,19 +5,27 @@ import { stringify } from "yaml";
 import { ContextKey } from "./ContextAggregator";
 import { slackFormatInstructions } from "../slackbot/utils";
 
-export const generateContextAnalysis = async (context: CheckContext[]) => {
-	const checkContext = stringify(
-		context.find((c) => c.key === "checkly.check")
+const getCheckContext = (context: CheckContext[]) => {
+	const checkContext = context.find((c) => c.key === ContextKey.ChecklyCheck);
+	return stringify(
+		{
+			checkId: checkContext?.checkId,
+			data: checkContext?.value,
+		},
+		{ indent: 2 }
 	);
+};
+
+export const generateContextAnalysis = async (context: CheckContext[]) => {
+	const checkContext = getCheckContext(context);
 
 	const generateContextAnalysis = async (text: string) => {
 		const prompt = `The following check has failed: ${checkContext}
 		
-		Analyze the following context and generate a concise summary to extract the most important context. Output only the relevant context summary, no other text.
+		Analyze the following context and generate a concise summary to extract the most important information. Output only the relevant context summary, no other text.
 
 CONTEXT:
-${text}
-`;
+${text}`;
 
 		const summary = await generateText({
 			model: getOpenaiSDKClient()("gpt-4o"),
@@ -40,13 +48,13 @@ ${text}
 };
 
 export const generateContextAnalysisSummary = async (
-	contextAnalysis: (CheckContext & { analysis: string })[]
+	contextAnalysis: CheckContext[]
 ) => {
+	const checkContext = getCheckContext(contextAnalysis);
 	const summary = await generateText({
 		model: getOpenaiSDKClient()("gpt-4o"),
-		prompt: `The following check has failed: ${stringify(
-			contextAnalysis.find((c) => c.key === ContextKey.ChecklyCheck)
-		)}
+		temperature: 1,
+		prompt: `The following check has failed: ${checkContext}
 	
 Anaylze the following context and generate a concise summary of the current situation.
 
@@ -61,14 +69,18 @@ OUTPUT FORMAT INSTRUCTIONS:
 ${slackFormatInstructions}
 
 CONTEXT:
-${contextAnalysis
-	.filter((c) => c.key !== ContextKey.ChecklyCheck)
-	.map((c) => `${c.key}: ${c.analysis}`)
-	.join("\n\n")}
+${stringify(
+	contextAnalysis
+		.filter((c) => c.key !== ContextKey.ChecklyCheck)
+		.map((c) => ({ key: c.key, source: c.source, value: c.value })),
+	{ indent: 2 }
+).slice(0, 200000)}
 
 Check-results amd checkly configuration details are already provided in the UI. Focus on the root cause analyisis and potential mitigations. Help the user to resolve the issue.
-Generate a condensed breakdown of the current situation. Focus on the essentials and provide a concise overview. Max. 100 words. Include links to relevant context if applicable.`,
-		maxTokens: 200,
+Generate a condensed summary of your root cause analysis of the current situation. Focus on the essentials, provide a concise overview and actionable insights. Provide reasoning and the source of the information. Max. 150 words. Include links to relevant context if applicable.
+
+*Summary:* `,
+		maxTokens: 500,
 	});
 
 	return summary.text;

src/aggregator/checkly-aggregator.ts

@@ -14,6 +14,7 @@ export const checklyAggregator = {
 			checkly.getCheck(alert.CHECK_ID),
 			checkly.getCheckResult(alert.CHECK_ID, alert.CHECK_RESULT_ID),
 		]);
+
 		const makeCheckContext = (key: ContextKey, value: unknown) => {
 			return {
 				checkId: alert.CHECK_ID,
@@ -27,7 +28,7 @@ export const checklyAggregator = {
 		const script = check.script;
 
 		const checklyCheckContext = [
-			makeCheckContext(ContextKey.ChecklyScript, script),
+			...(script ? [makeCheckContext(ContextKey.ChecklyScript, script)] : []),
 			makeCheckContext(ContextKey.ChecklyCheck, mapCheckToContextValue(check)),
 			makeCheckContext(
 				ContextKey.ChecklyResults,

src/aggregator/github-aggregator.ts

@@ -1,7 +1,18 @@
 import GitHubAPI from "../github/github";
 import { WebhookAlertDto } from "../checkly/alertDTO";
 import { CheckContext, ContextKey } from "./ContextAggregator";
-import moment from "moment";
+import {
+	getLastSuccessfulCheckResult,
+	mapCheckResultToContextValue,
+	mapCheckToContextValue,
+} from "../../src/checkly/utils";
+import { prisma } from "../../src/prisma";
+import { generateObject } from "ai";
+import { getOpenaiSDKClient } from "../../src/ai/openai";
+import { checkly } from "../../src/checkly/client";
+import { stringify } from "yaml";
+import { z } from "zod";
+import { Release } from "@prisma/client";
 
 const githubApi = new GitHubAPI(process.env.CHECKLY_GITHUB_TOKEN || "");
 
@@ -26,54 +37,54 @@ interface RepoChange {
 	}>;
 }
 
-async function getRecentChanges(repo: string): Promise<RepoChange> {
-	const [owner, repoName] = repo.split("/");
-	const since = moment().subtract(24, "hours").toISOString();
-
-	// Get recent commits
-	const commits = await githubApi
-		.getCommits(owner, repoName, { since })
-		.catch((error) => {
-			console.error("Error fetching commits:", error);
-			return [];
-		});
-
-	// Get recent pull requests
-	const pullRequests = await githubApi.getPullRequests(owner, repoName);
-
-	// Get recent releases
-	const releases = await githubApi.queryLatestReleasesWithDiffs(
-		owner,
-		repoName,
-		new Date(since)
-	);
-
-	// Filter PRs updated in the last 24 hours
-	const recentPRs = pullRequests.filter((pr) =>
-		moment(pr.updated_at).isAfter(moment().subtract(24, "hours"))
-	);
-
-	return {
-		repo,
-		commits: commits.map((commit) => ({
-			sha: commit.sha,
-			message: commit.commit.message,
-			author: commit.commit.author?.name || "Unknown",
-			date: commit.commit.author?.date || "",
-		})),
-		pullRequests: recentPRs.map((pr) => ({
-			number: pr.number,
-			title: pr.title,
-			state: pr.state,
-			author: pr.user?.login || "Unknown",
-			url: pr.html_url,
-		})),
-		releases: releases.map((release) => ({
-			release: release.release,
-			diff: release.diff,
-		})),
-	};
-}
+// async function getRecentChanges(repo: string): Promise<RepoChange> {
+// 	const [owner, repoName] = repo.split("/");
+// 	const since = moment().subtract(24, "hours").toISOString();
+
+// 	// Get recent commits
+// 	const commits = await githubApi
+// 		.getCommits(owner, repoName, { since })
+// 		.catch((error) => {
+// 			console.error("Error fetching commits:", error);
+// 			return [];
+// 		});
+
+// 	// Get recent pull requests
+// 	const pullRequests = await githubApi.getPullRequests(owner, repoName);
+
+// 	// Get recent releases
+// 	const releases = await githubApi.queryLatestReleasesWithDiffs(
+// 		owner,
+// 		repoName,
+// 		new Date(since)
+// 	);
+
+// 	// Filter PRs updated in the last 24 hours
+// 	const recentPRs = pullRequests.filter((pr) =>
+// 		moment(pr.updated_at).isAfter(moment().subtract(24, "hours"))
+// 	);
+
+// 	return {
+// 		repo,
+// 		commits: commits.map((commit) => ({
+// 			sha: commit.sha,
+// 			message: commit.commit.message,
+// 			author: commit.commit.author?.name || "Unknown",
+// 			date: commit.commit.author?.date || "",
+// 		})),
+// 		pullRequests: recentPRs.map((pr) => ({
+// 			number: pr.number,
+// 			title: pr.title,
+// 			state: pr.state,
+// 			author: pr.user?.login || "Unknown",
+// 			url: pr.html_url,
+// 		})),
+// 		releases: releases.map((release) => ({
+// 			release: release.release,
+// 			diff: release.diff,
+// 		})),
+// 	};
+// }
 
 export const githubAggregator = {
 	name: "GitHub",
@@ -82,41 +93,75 @@ export const githubAggregator = {
 		try {
 			await githubApi.checkRateLimit();
 
-			const REPOS = process.env.GITHUB_REPOS
-				? JSON.parse(process.env.GITHUB_REPOS)
-				: [];
+			const lastSuccessfulCheckResult = await getLastSuccessfulCheckResult(
+				alert.CHECK_ID
+			);
 
-			// If no repos configured, try to get all repos from the organization
-			if (REPOS.length === 0 && process.env.GITHUB_ORG) {
-				const repos = await githubApi.queryRepositories(process.env.GITHUB_ORG);
-				REPOS.push(
-					...repos.map((repo) => `${process.env.GITHUB_ORG}/${repo.name}`)
-				);
-			}
+			const failureResults = await checkly.getCheckResult(
+				alert.CHECK_ID,
+				alert.CHECK_RESULT_ID
+			);
 
-			// Fetch changes for all configured repositories
-			const repoChanges = await Promise.all(
-				REPOS.map((repo) => getRecentChanges(repo))
+			const check = await checkly.getCheck(alert.CHECK_ID);
+
+			const releases = await prisma.release.findMany({
+				where: {
+					publishedAt: {
+						gte: new Date(lastSuccessfulCheckResult.startedAt),
+					},
+				},
+			});
+
+			const { object: relevantReleaseIds } = await generateObject({
+				model: getOpenaiSDKClient()("gpt-4o"),
+				prompt: `Based on the following releases, which ones are most relevant to the check failure? Analyze the check script, result and releases to determine which releases are most relevant. Provide a list of release ids that are most relevant to the check.
+
+Releases:
+${stringify(
+	releases.map((r) => ({
+		id: r.id,
+		repo: r.repoUrl,
+		release: r.name,
+		summary: r.summary,
+	}))
+)}
+
+Check:
+${stringify(mapCheckToContextValue(check))}
+
+Check Script:
+${check.script}
+
+Check Result:
+${stringify(mapCheckResultToContextValue(failureResults))}`,
+				schema: z.object({
+					releaseIds: z
+						.array(z.string())
+						.describe(
+							"The ids of the releases that are most relevant to the check failure."
+						),
+				}),
+			});
+
+			const relevantReleases = releases.filter((r) =>
+				relevantReleaseIds.releaseIds.includes(r.id)
 			);
 
-			const makeRepoChangesContext = (repoChange: RepoChange) =>
+			const makeRepoReleaseContext = (release: Release) =>
 				({
-					key: ContextKey.GitHubRepoChanges.replace("$repo", repoChange.repo),
-					value: repoChange,
+					key: ContextKey.GitHubReleaseSummary.replace(
+						"$repo",
+						`${release.org}/${release.repo}`
+					),
+					value: release,
 					checkId: alert.CHECK_ID,
 					source: "github",
 				} as CheckContext);
 
-			if (repoChanges) {
-				const context = repoChanges
-					.filter(
-						(repoChange) =>
-							repoChange.commits.length > 0 ||
-							repoChange.pullRequests.length > 0 ||
-							repoChange.releases.length > 0
-					)
-					.map((repoChange) => makeRepoChangesContext(repoChange));
-
+			if (relevantReleases) {
+				const context = relevantReleases.map((release) =>
+					makeRepoReleaseContext(release)
+				);
 				return context;
 			}
 

src/checkly/utils.ts

@@ -1,3 +1,4 @@
+import { checkly } from "./client";
 import { Check, CheckResult } from "./models";
 
 export const mapCheckToContextValue = (check: Check) => {
@@ -31,3 +32,8 @@ export const mapCheckResultToContextValue = (result: CheckResult) => {
 		resultType: result.resultType,
 	};
 };
+
+export const getLastSuccessfulCheckResult = async (checkId: string) => {
+	const results = await checkly.getCheckResults(checkId, false, 1);
+	return results[0];
+};