JACoB PR for Issue Enhance the Jira to GitHub issue conversion

src/app/dashboard/[org]/[repo]/todos/Todo.tsx

@@ -13,6 +13,7 @@ import TodoDetailsPlaceholder from "~/app/_components/DetailsPlaceholder";
 import { useRouter } from "next/navigation";
 
 const MOBILE_WIDTH_BREAKPOINT = 768;
+import { type Todo as TodoType } from "~/server/api/routers/events";
 
 export interface Issue {
   title: string;
@@ -178,6 +179,13 @@ const Todo: React.FC<TodoProps> = ({ org, repo }) => {
 
       {/* Details column: Selected todo details */}
       <div className="hide-scrollbar hidden h-[calc(100vh-116px)] overflow-y-scroll bg-white p-6 dark:bg-gray-800 md:block md:w-2/3">
+        {selectedTodo?.evaluationScore && selectedTodo.evaluationScore < 4 ? (
+          <div className="mb-4 rounded-md border border-yellow-400 bg-yellow-50 p-4 text-yellow-800">
+            <h3 className="text-lg font-semibold">Feedback:</h3>
+            <p>{selectedTodo.feedback}</p>
+          </div>
+        ) : null}
+
         {selectedTodo ? (
           <IssueDetails
             selectedTodo={selectedTodo}

src/server/db/migrations/20241130000000_updateIssuesTable.ts

@@ -0,0 +1,23 @@
+import { type change } from "postgres-migrations";
+
+export async function up(pgm: change) {
+  pgm.addColumns("issues", {
+    jiraIssueDescription: {
+      type: "text",
+      notNull: false,
+    },
+    evaluationScore: {
+      type: "numeric",
+      notNull: false,
+    },
+    feedback: {
+      type: "text",
+      notNull: false,
+    },
+    didCreateGithubIssue: {
+      type: "boolean",
+      notNull: true,
+      default: false,
+    },
+  });
+}

src/server/db/tables/issues.table.ts

@@ -9,6 +9,10 @@ export class IssuesTable extends BaseTable {
     }),
     issueId: t.varchar(255),
     title: t.text().nullable(),
+    jiraIssueDescription: t.text().nullable(),
+    evaluationScore: t.numeric().nullable(),
+    feedback: t.text().nullable(),
+    didCreateGithubIssue: t.boolean().default(false),
     ...t.timestamps(),
   }));
 }

src/server/utils/evaluateIssue.test.ts

@@ -3,6 +3,7 @@ import { evaluateIssue } from "./evaluateIssue";
 import * as openaiRequest from "../openai/request";
 import { PlanningAgentActionType } from "../db/enums";
 import { type StandardizedPath } from "./files";
+import { evaluateJiraIssue } from "./evaluateIssue";
 
 vi.mock("../openai/request", () => ({
   sendGptRequestWithSchema: vi.fn(),
@@ -248,4 +249,31 @@ describe("evaluateIssue", () => {
       "claude-3-5-sonnet-20241022",
     );
   });
+
+  it("evaluateJiraIssue should return appropriate evaluation", async () => {
+    const mockJiraEvaluation = {
+      evaluationScore: 3.5,
+      feedback: "The issue description lacks sufficient detail.",
+    };
+
+    vi.mocked(openaiRequest.sendGptRequestWithSchema).mockResolvedValue(
+      mockJiraEvaluation,
+    );
+
+    const result = await evaluateJiraIssue(
+      "Implement authentication",
+      "We need to add login functionality.",
+    );
+
+    expect(result).toEqual(mockJiraEvaluation);
+    expect(openaiRequest.sendGptRequestWithSchema).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.any(String),
+      expect.any(Object),
+      0.4,
+      undefined,
+      3,
+      "claude-3-5-sonnet-20241022",
+    );
+  });
 });

src/server/utils/evaluateIssue.ts

@@ -5,6 +5,13 @@ import { type ContextItem } from "./codebaseContext";
 import { type PlanStep } from "../db/tables/planSteps.table";
 import { PlanningAgentActionType } from "../db/enums";
 
+const JiraEvaluationSchema = z.object({
+  evaluationScore: z.number().min(1).max(5),
+  feedback: z.string().optional(),
+});
+
+export type JiraEvaluation = z.infer<typeof JiraEvaluationSchema>;
+
 const EvaluationSchema = z.object({
   confidenceScore: z.number().min(0).max(5),
   complexityFactors: z.object({
@@ -150,3 +157,45 @@ Please provide your evaluation in the following JSON format:
 
   return evaluation;
 }
+
+export async function evaluateJiraIssue(
+  title: string,
+  description: string,
+): Promise<JiraEvaluation> {
+  const systemPrompt = `You are an expert software architect and technical evaluator. Your task is to analyze the given Jira issue and provide an evaluation of its quality and readiness for an AI coding agent to address it.
+
+Consider the following when evaluating:
+- Clarity and specificity of the issue title and description.
+- Completeness of the information required to implement the task.
+- Whether the requirements are actionable without ambiguity.
+
+Provide a concise evaluation score and feedback message if necessary.`;
+
+  const userPrompt = `Jira Issue Title: "${title}"
+
+Jira Issue Description: "${description}"
+
+Based on the above, provide:
+
+- An evaluation score between 1 and 5 (half-points acceptable), indicating how likely it is that an AI coding agent can flawlessly complete the task.
+
+- If the score is less than 4, provide a one-sentence feedback message informing the user what needs to be changed to make the ticket actionable by JACoB.
+
+Provide the response in the following JSON format:
+{
+  "evaluationScore": number,
+  "feedback": string // optional, include only if score is less than 4
+}`;
+
+  const evaluation = await sendGptRequestWithSchema(
+    userPrompt,
+    systemPrompt,
+    JiraEvaluationSchema,
+    0.4,
+    undefined,
+    3,
+    "claude-3-5-sonnet-20241022",
+  );
+
+  return evaluation;
+}

src/server/utils/jira.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect, vi } from "vitest";
+import { fetchNewJiraIssues } from "./jira";
+import * as evaluateIssueModule from "./evaluateIssue";
+import * as dbModule from "~/server/db/db";
+
+vi.mock("./evaluateIssue", () => ({
+  evaluateJiraIssue: vi.fn(),
+}));
+
+vi.mock("~/server/db/db", () => ({
+  db: {
+    issues: {
+      findByOptional: vi.fn(),
+      create: vi.fn(),
+    },
+    issueBoards: {
+      findBy: vi.fn(),
+    },
+    projects: {
+      findBy: vi.fn(),
+    },
+  },
+}));
+
+describe("fetchNewJiraIssues", () => {
+  it("should not create GitHub issue if evaluation score is less than 4", async () => {
+    const mockEvaluation = {
+      evaluationScore: 3.5,
+      feedback: "The issue description lacks sufficient detail.",
+    };
+
+    vi.mocked(evaluateIssueModule.evaluateJiraIssue).mockResolvedValue(
+      mockEvaluation,
+    );
+
+    // Mock other dependencies and API responses as needed
+
+    // Call fetchNewJiraIssues with necessary parameters
+    await fetchNewJiraIssues({
+      jiraAccessToken: "fake-token",
+      cloudId: "cloud-id",
+      projectId: 1,
+      boardId: "board-id",
+      userId: 1,
+      githubAccessToken: "github-token",
+    });
+
+    // Expectations
+    expect(evaluateIssueModule.evaluateJiraIssue).toHaveBeenCalled();
+    expect(dbModule.db.issues.create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        evaluationScore: 3.5,
+        feedback: "The issue description lacks sufficient detail.",
+        didCreateGithubIssue: false,
+      }),
+    );
+    // Ensure createGitHubIssue was not called
+  });
+
+  it("should create GitHub issue if evaluation score is 4 or higher", async () => {
+    const mockEvaluation = {
+      evaluationScore: 4.0,
+      feedback: null,
+    };
+
+    vi.mocked(evaluateIssueModule.evaluateJiraIssue).mockResolvedValue(
+      mockEvaluation,
+    );
+
+    // Mock other dependencies and API responses as needed
+
+    // Call fetchNewJiraIssues with necessary parameters
+    await fetchNewJiraIssues({
+      jiraAccessToken: "fake-token",
+      cloudId: "cloud-id",
+      projectId: 1,
+      boardId: "board-id",
+      userId: 1,
+      githubAccessToken: "github-token",
+    });
+
+    // Expectations
+    expect(evaluateIssueModule.evaluateJiraIssue).toHaveBeenCalled();
+    expect(dbModule.db.issues.create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        evaluationScore: 4.0,
+        feedback: null,
+        didCreateGithubIssue: true,
+      }),
+    );
+    // Ensure createGitHubIssue was called
+  });
+});

src/server/utils/jira.ts

@@ -13,6 +13,7 @@ import { refreshGitHubAccessToken } from "../github/tokens";
 import { createGitHubIssue, rewriteGitHubIssue } from "../github/issue";
 import { uploadToS3, getSignedUrl, IMAGE_TYPE } from "../utils/images";
 const bucketName = process.env.BUCKET_NAME ?? "";
+import { evaluateJiraIssue } from "./evaluateIssue";
 
 export async function refreshJiraAccessToken(
   accountId: number,
@@ -336,15 +337,34 @@ export async function fetchNewJiraIssues({
         continue;
       }
 
-      console.log(
-        `Repo ${project.repoFullName}: Creating new Jira issue ${issue.id}`,
+      // Evaluate the Jira issue
+      const evaluation = await evaluateJiraIssue(
+        issue.title,
+        issue.description,
       );
+
+      // Save the issue with evaluation data
       await db.issues.create({
         issueBoardId: issueBoard.id,
         issueId: issue.id,
         title: issue.title,
+        jiraIssueDescription: issue.description,
+        evaluationScore: evaluation.evaluationScore,
+        feedback: evaluation.feedback ?? null,
+        didCreateGithubIssue: evaluation.evaluationScore >= 4,
       });
 
+      if (evaluation.evaluationScore < 4) {
+        console.log(
+          `Repo ${project.repoFullName}: Jira issue ${issue.id} did not pass evaluation (score: ${evaluation.evaluationScore}). Not creating GitHub issue.`,
+        );
+        continue;
+      }
+
+      console.log(
+        `Repo ${project.repoFullName}: Creating GitHub issue for Jira issue ${issue.id}`,
+      );
+
       const owner = project.repoFullName.split("/")[0];
       const repo = project.repoFullName.split("/")[1];
       if (!owner || !repo) {