fix: slack message format + update prompts

src/aggregator/chains.ts

@@ -3,6 +3,7 @@ import { getOpenaiSDKClient } from "../ai/openai";
 import { CheckContext } from "../aggregator/ContextAggregator";
 import { stringify } from "yaml";
 import { ContextKey } from "./ContextAggregator";
+import { slackFormatInstructions } from "../slackbot/utils";
 
 export const generateContextAnalysis = async (context: CheckContext[]) => {
 	const checkContext = stringify(
@@ -12,7 +13,7 @@ export const generateContextAnalysis = async (context: CheckContext[]) => {
 	const generateContextAnalysis = async (text: string) => {
 		const prompt = `The following check has failed: ${checkContext}
 		
-		Analyze the following context and generate a dense summary of the current situation.
+		Analyze the following context and generate a concise summary to extract the most important context. Output only the relevant context summary, no other text.
 
 CONTEXT:
 ${text}
@@ -21,7 +22,7 @@ ${text}
 		const summary = await generateText({
 			model: getOpenaiSDKClient()("gpt-4o"),
 			prompt: prompt,
-			temperature: 0.5,
+			temperature: 0.1,
 			maxTokens: 300,
 		});
 
@@ -47,23 +48,26 @@ export const generateContextAnalysisSummary = async (
 			contextAnalysis.find((c) => c.key === ContextKey.ChecklyCheck)
 		)}
 	
-Anaylze the following context and generate a dense summary of the current situation.
+Anaylze the following context and generate a concise summary of the current situation.
 
-*CONSTITUTION:*
+CONSTITUTION:
 - Always prioritize accuracy and relevance in your insights and recommendations
 - Be concise but comprehensive in your explanations
 - Focus on providing actionable information that can help reduce MTTR
 - The user is a experienced devops engineer. Don't overcomplicate it, focus on the context and provide actionable insights. They know what they are doing, don't worry about the details.
 - Don't include the check configuration or run details, focus on logs, changes and the current state of the system.
 
-*CONTEXT:*
+OUTPUT FORMAT INSTRUCTIONS:
+${slackFormatInstructions}
+
+CONTEXT:
 ${contextAnalysis
 	.filter((c) => c.key !== ContextKey.ChecklyCheck)
-	.map((c) => c.analysis)
+	.map((c) => `${c.key}: ${c.analysis}`)
 	.join("\n\n")}
 
-
-Generate a condensed breakdown of the current situation. Focus on the essentials and provide a concise overview. Max. 100 words. Format your responses as a slack message (*bold*, _italic_, ~strikethrough~, <http://www.example.com|This *is* a link>) and keep the answer concise and relevant. Include links (slack format e.g. <https://example.com|Example>) to the relevant context in your response if applicable.`,
+Check-results amd checkly configuration details are already provided in the UI. Focus on the root cause analyisis and potential mitigations. Help the user to resolve the issue.
+Generate a condensed breakdown of the current situation. Focus on the essentials and provide a concise overview. Max. 100 words. Include links to relevant context if applicable.`,
 		maxTokens: 200,
 	});
 

src/ai/Assistant.ts

@@ -172,19 +172,11 @@ export class BaseAssistant {
 	private async runTool(
 		toolCall: RequiredActionFunctionToolCall
 	): Promise<unknown> {
-		console.log(
-			"Running tool",
-			toolCall.function.name,
-			toolCall.function.arguments
-		);
-
 		try {
 			const parameters = JSON.parse(toolCall.function.arguments);
 			const tool = await this.getTool(toolCall.function.name);
 			const output = await tool.run(parameters);
 
-			console.log("Tool output: ", output);
-
 			this.runContext?.toolCallStack.push({ ...toolCall, output });
 			return output;
 		} catch (error) {

src/routes/checklywebhook.ts

@@ -2,7 +2,10 @@ import express, { Request, Response, NextFunction, text } from "express";
 import { plainToInstance } from "class-transformer";
 import { validateOrReject } from "class-validator";
 import "reflect-metadata";
-import { CheckContextAggregator } from "../aggregator/ContextAggregator";
+import {
+	CheckContextAggregator,
+	ContextKey,
+} from "../aggregator/ContextAggregator";
 import {
 	generateContextAnalysis,
 	generateContextAnalysisSummary,
@@ -50,7 +53,10 @@ router.post("/", async (req: Request, res: Response) => {
 			},
 		});
 
-		if (exisingAlert && process.env.ALLOW_DUPLICATE_ALERTS !== "true") {
+		if (
+			(exisingAlert && !process.env.ALLOW_DUPLICATE_ALERTS) ||
+			process.env.ALLOW_DUPLICATE_ALERTS !== "true"
+		) {
 			res.status(200).json({ message: "Alert already processed" });
 		} else {
 			const aggregator = new CheckContextAggregator(alertDto);
@@ -76,6 +82,10 @@ router.post("/", async (req: Request, res: Response) => {
 				},
 			});
 
+			const checkResults = context.find(
+				(c) => c.key === ContextKey.ChecklyResults
+			);
+
 			const thread = await getOpenaiClient().beta.threads.create({
 				messages: [
 					{
@@ -114,29 +124,43 @@ router.post("/", async (req: Request, res: Response) => {
 						fields: [
 							{
 								type: "mrkdwn",
-								text: `🩺 *Check:* <https://app.checklyhq.com/checks/${alertDto.CHECK_ID}|${alertDto.CHECK_NAME}>`,
+								text: `:checkly-hyped-2: *Check*\n<https://app.checklyhq.com/checks/${alertDto.CHECK_ID}|${alertDto.CHECK_NAME}>`,
 							},
 							{
 								type: "mrkdwn",
-								text: `🔮 *Result:* <${alertDto.RESULT_LINK}|View>`,
+								text: `:crystal_ball: *Result*\n<${alertDto.RESULT_LINK}|Open>`,
 							},
 							{
 								type: "mrkdwn",
-								text: `📅 *When:* ${new Date(
+								text: `:date: *Date*\n${new Date(
 									alertDto.STARTED_AT
 								).toLocaleString()}`,
 							},
 							{
 								type: "mrkdwn",
-								text: `🌍 *Location:* ${alertDto.RUN_LOCATION}`,
+								text: `:globe_with_meridians: *Location*\n${alertDto.RUN_LOCATION}`,
+							},
+							{
+								type: "mrkdwn",
+								text: `:stopwatch: *Response Time*\n${
+									(checkResults?.value as any).responseTime
+										? (checkResults?.value as any).responseTime + "ms"
+										: "unknown"
+								}`,
+							},
+							{
+								type: "mrkdwn",
+								text: `:recycle: *Attempts*\n${
+									(checkResults?.value as any).attempts ?? "unknown"
+								}`,
 							},
 						],
 					},
 					{
 						type: "section",
 						text: {
 							type: "mrkdwn",
-							text: `*Summary*\n${summary}`,
+							text: `${summary}`,
 						},
 					},
 				],
@@ -149,26 +173,36 @@ router.post("/", async (req: Request, res: Response) => {
 			});
 
 			await assistant.addMessage(
-				`What happened? Check *GitHub* & Investigate further...`
+				`Investigate further. Use the following approach:
+- Check if other checks are failing. Is there a related incident?
+- Identify relevant github repositories that could be causing the issue
+- Check the github activity in the identified repositories
+- Try to identify the root cause of the issue or at least a good starting point
+
+Continue using your tools extensively to get a holistic view of the situation.
+Keep in mind, we are investigating check "${alertDto.CHECK_NAME}"`
 			);
 
 			const run = await assistant.runSync();
 			const responseMessages = await getRunMessages(thread.id, run.id);
 
+			const messages = responseMessages.map((msg) =>
+				msg.content
+					.filter((c) => c.type === "text")
+					.map((c) => (c as any).text.value)
+					.join("")
+			);
+
 			// Post assistant's analysis to the thread
-			await Promise.all(
-				responseMessages.map((msg) =>
+			const response = await Promise.all(
+				messages.map(async (msg) =>
 					app.client.chat.postMessage({
 						channel: process.env.SLACK_ALERT_CHANNEL_ID as string,
 						thread_ts: oMessage.ts,
-						text: msg.content
-							.filter((c) => c.type === "text")
-							.map((c) => (c as any).text.value)
-							.join(""),
+						text: msg,
 						metadata: {
 							event_type: "alert",
 							event_payload: {
-								alertId: alert.id,
 								threadId: thread.id,
 							},
 						},

src/slackbot/app.ts

@@ -127,7 +127,7 @@ app.event("app_mention", async ({ event, context }) => {
 			app.client.chat.postMessage({
 				token: context.botToken,
 				channel: event.channel,
-				text: msg,
+				text: msg.replace(/(\*\*|__)(.*?)\1/g, "*$2*"),
 				thread_ts: threadTs,
 				...(threadId && {
 					metadata: {