WIP 2.0.0: message drafts use event placeholders only with an event

Found by the first real eval run (Sonnet 5.5 through the local Hub,
19/19 checks): without an event the drafts still used {{event.title}}
and added an event-details block. The instructions now offer those
placeholders only when an event is given; the eval checks for it, and
adds up split themes when counting feedback. Route tests for the five
AI helpers in v2.test.ts; the eval script loads backend/.env.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ls1vMeckMWrKPfc1MMyMJF
This commit is contained in:
2026-10-30 18:25:57 +02:00
co-authored by Claude Opus 5.5
parent 4924d1ed76
commit ba121118ea
4 changed files with 105 additions and 5 deletions
+5 -1
View File
@@ -10,6 +10,7 @@
* The examples come from the October 2026 model trial; the inputs are made up and hold no
* personal information.
*/
import 'dotenv/config';
import { prisma } from '../src/v2/legacy';
import { draftEvent, summariseReport, draftMessage, summariseFeedback, improveText } from '../src/ai/helpers';
@@ -83,7 +84,8 @@ const CASES: Case[] = [
{ label: 'Food', type: 'rating', answered: 108, average: 3.6 }, { label: 'Comments', type: 'text', answered: FEEDBACK_COMMENTS.length, texts: FEEDBACK_COMMENTS },
],
});
const count = (re: RegExp) => summary.improve.find(t => re.test(t.theme))?.mentions ?? -1;
// A theme may be split (e.g. "lunch queues" and "vegetarian food"): add up every match.
const count = (re: RegExp) => summary.improve.filter(t => re.test(t.theme)).reduce((n, t) => n + t.mentions, 0);
return {
output: summary,
checks: [
@@ -104,6 +106,7 @@ const CASES: Case[] = [
['uses {{name}}', message.body.includes('{{name}}')],
['keeps 7:30, sleeping bag, car park', /7:30/.test(message.body) && /sleeping bag/i.test(message.body) && /car park/i.test(message.body)],
['no "reminda" (seen with Haiku)', !/reminda/i.test(message.body)],
['no event placeholders without an event', !/\{\{event\./.test(message.body)],
],
};
},
@@ -118,6 +121,7 @@ const CASES: Case[] = [
['has a subject', message.subject.trim().length > 5],
['keeps the facts', /Protea/.test(message.body) && /7:00/.test(message.body) && /R120/.test(message.body)],
['the email address came back intact', message.body.includes('events@church.example.org')],
['no event placeholders without an event', !/\{\{event\./.test(`${message.subject} ${message.body}`)],
],
};
},