#!/usr/bin/env bun
/**
* Accuracy check for AI merge fixes on realistic conflicts, meant for a dev server using the real models
* (MERGEFIX_PROVIDER=workers-ai). For each case it reports whether the top-ranked resolution is correct,
* whether it was auto-applied, and Clef's probabilities, so the gating can be judged against reality.
*
* BASE_URL=http://localhost:3015 D1_CWD=$PWD bun scripts/eval-mergefix.ts
*/
import { api, d1, scenario } from "./mergefix-harness";
import type { ConflictFile } from "./mergefix-harness";
type Expectation = {
name: string;
pr: { title: string; body: string };
files: ConflictFile[];
/** True when the resolved file is semantically right. */
correct: (resolved: string) => boolean;
/** Whether an ideal system would auto-apply this one. */
shouldAutoApply: boolean;
};
const CASES: Expectation[] = [
{
name: "both sides add an import",
pr: { title: "Add date formatting to the report", body: "Uses formatDate in the report header." },
files: [
{
path: "src/report.ts",
base: 'import { sum } from "./math";\n\nexport function report(values: number[]) {\n return `total ${sum(values)}`;\n}\n',
head: 'import { sum } from "./math";\nimport { formatDate } from "./dates";\n\nexport function report(values: number[]) {\n return `total ${sum(values)}`;\n}\n',
main: 'import { sum } from "./math";\nimport { logger } from "./log";\n\nexport function report(values: number[]) {\n return `total ${sum(values)}`;\n}\n',
},
],
correct: (text) => text.includes("formatDate") && text.includes("logger"),
shouldAutoApply: true,
},
{
name: "both sides append a changelog entry",
pr: { title: "Add CSV export", body: "Adds a CSV export button." },
files: [