WIP 2.0.0: message drafts use event placeholders only with an event
Found by the first real eval run (Sonnet 5.5 through the local Hub,
19/19 checks): without an event the drafts still used {{event.title}}
and added an event-details block. The instructions now offer those
placeholders only when an event is given; the eval checks for it, and
adds up split themes when counting feedback. Route tests for the five
AI helpers in v2.test.ts; the eval script loads backend/.env.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ls1vMeckMWrKPfc1MMyMJF
This commit is contained in:
@@ -10,6 +10,7 @@
|
||||
* The examples come from the October 2026 model trial; the inputs are made up and hold no
|
||||
* personal information.
|
||||
*/
|
||||
import 'dotenv/config';
|
||||
import { prisma } from '../src/v2/legacy';
|
||||
import { draftEvent, summariseReport, draftMessage, summariseFeedback, improveText } from '../src/ai/helpers';
|
||||
|
||||
@@ -83,7 +84,8 @@ const CASES: Case[] = [
|
||||
{ label: 'Food', type: 'rating', answered: 108, average: 3.6 }, { label: 'Comments', type: 'text', answered: FEEDBACK_COMMENTS.length, texts: FEEDBACK_COMMENTS },
|
||||
],
|
||||
});
|
||||
const count = (re: RegExp) => summary.improve.find(t => re.test(t.theme))?.mentions ?? -1;
|
||||
// A theme may be split (e.g. "lunch queues" and "vegetarian food"): add up every match.
|
||||
const count = (re: RegExp) => summary.improve.filter(t => re.test(t.theme)).reduce((n, t) => n + t.mentions, 0);
|
||||
return {
|
||||
output: summary,
|
||||
checks: [
|
||||
@@ -104,6 +106,7 @@ const CASES: Case[] = [
|
||||
['uses {{name}}', message.body.includes('{{name}}')],
|
||||
['keeps 7:30, sleeping bag, car park', /7:30/.test(message.body) && /sleeping bag/i.test(message.body) && /car park/i.test(message.body)],
|
||||
['no "reminda" (seen with Haiku)', !/reminda/i.test(message.body)],
|
||||
['no event placeholders without an event', !/\{\{event\./.test(message.body)],
|
||||
],
|
||||
};
|
||||
},
|
||||
@@ -118,6 +121,7 @@ const CASES: Case[] = [
|
||||
['has a subject', message.subject.trim().length > 5],
|
||||
['keeps the facts', /Protea/.test(message.body) && /7:00/.test(message.body) && /R120/.test(message.body)],
|
||||
['the email address came back intact', message.body.includes('events@church.example.org')],
|
||||
['no event placeholders without an event', !/\{\{event\./.test(`${message.subject} ${message.body}`)],
|
||||
],
|
||||
};
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user