WIP 2.0.0: message drafts use event placeholders only with an event

Found by the first real eval run (Sonnet 5.5 through the local Hub,
19/19 checks): without an event the drafts still used {{event.title}}
and added an event-details block. The instructions now offer those
placeholders only when an event is given; the eval checks for it, and
adds up split themes when counting feedback. Route tests for the five
AI helpers in v2.test.ts; the eval script loads backend/.env.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ls1vMeckMWrKPfc1MMyMJF
This commit is contained in:
2026-10-30 18:25:57 +02:00
co-authored by Claude Opus 5.5
parent 4924d1ed76
commit ba121118ea
4 changed files with 105 additions and 5 deletions
+5 -1
View File
@@ -10,6 +10,7 @@
* The examples come from the October 2026 model trial; the inputs are made up and hold no
* personal information.
*/
import 'dotenv/config';
import { prisma } from '../src/v2/legacy';
import { draftEvent, summariseReport, draftMessage, summariseFeedback, improveText } from '../src/ai/helpers';
@@ -83,7 +84,8 @@ const CASES: Case[] = [
{ label: 'Food', type: 'rating', answered: 108, average: 3.6 }, { label: 'Comments', type: 'text', answered: FEEDBACK_COMMENTS.length, texts: FEEDBACK_COMMENTS },
],
});
const count = (re: RegExp) => summary.improve.find(t => re.test(t.theme))?.mentions ?? -1;
// A theme may be split (e.g. "lunch queues" and "vegetarian food"): add up every match.
const count = (re: RegExp) => summary.improve.filter(t => re.test(t.theme)).reduce((n, t) => n + t.mentions, 0);
return {
output: summary,
checks: [
@@ -104,6 +106,7 @@ const CASES: Case[] = [
['uses {{name}}', message.body.includes('{{name}}')],
['keeps 7:30, sleeping bag, car park', /7:30/.test(message.body) && /sleeping bag/i.test(message.body) && /car park/i.test(message.body)],
['no "reminda" (seen with Haiku)', !/reminda/i.test(message.body)],
['no event placeholders without an event', !/\{\{event\./.test(message.body)],
],
};
},
@@ -118,6 +121,7 @@ const CASES: Case[] = [
['has a subject', message.subject.trim().length > 5],
['keeps the facts', /Protea/.test(message.body) && /7:00/.test(message.body) && /R120/.test(message.body)],
['the email address came back intact', message.body.includes('events@church.example.org')],
['no event placeholders without an event', !/\{\{event\./.test(`${message.subject} ${message.body}`)],
],
};
},
+5 -3
View File
@@ -216,13 +216,15 @@ export type MessageAnswer = { subject: string; body: string };
const MESSAGE_SCHEMA = obj({ subject: str, body: str }, { subject: 'Empty for WhatsApp' });
const MESSAGE_SYSTEM = (channel: 'email' | 'whatsapp', now: Date) => [
const MESSAGE_SYSTEM = (channel: 'email' | 'whatsapp', now: Date, withEvent: boolean) => [
'You write messages from a church events team to people registered for an event, from the organiser\'s short note.',
todayLine(now),
channel === 'whatsapp'
? '- WhatsApp: plain text, at most 3 short sentences, at most one emoji. "subject" is empty.'
: '- Email: a clear subject, then the body in short plain-text paragraphs, the most important thing first. Start with "Hi {{name}}," and end with "The events team".',
'- Use {{name}} for the person\'s first name. {{event.title}}, {{event.start}} and {{event.link}} are filled in for each event, if useful.',
withEvent
? '- Use {{name}} for the person\'s first name. {{event.title}}, {{event.start}} and {{event.link}} are filled in with the event\'s details; use them only where the message needs them, and don\'t add a separate block of event details.'
: '- Use {{name}} for the person\'s first name. Use no other {{…}} placeholders: name the event as the note does.',
'- Use only the facts in the note and the event details; never invent times, places, prices, links or contact details, or add advice the note doesn\'t give.',
'- Spell carefully: this goes to everyone registered.',
KEEP_PLACEHOLDERS,
@@ -234,7 +236,7 @@ export async function draftMessage(userId: string, input: { channel: 'email' | '
const p = protect(input.note);
const facts = input.event ? `\n\nEvent: ${input.event.title}, ${input.event.start}${input.event.location ? `, at ${input.event.location}` : ''}` : '';
const { output, status } = await runHelper('draft_message', userId, {
system: MESSAGE_SYSTEM(input.channel, now), input: `Note: ${p.text}${facts}`, schema: MESSAGE_SCHEMA, maxTokens: input.channel === 'whatsapp' ? 600 : 1500, effort: 'low',
system: MESSAGE_SYSTEM(input.channel, now, !!input.event), input: `Note: ${p.text}${facts}`, schema: MESSAGE_SCHEMA, maxTokens: input.channel === 'whatsapp' ? 600 : 1500, effort: 'low',
});
const message = restore(output as MessageAnswer, p.found);
return { message: input.channel === 'whatsapp' ? { subject: '', body: message.body } : message, status };
@@ -110,6 +110,15 @@ describe('what the helpers send', () => {
expect(r.message).toEqual({ subject: '', body: 'Hi {{name}}!' });
expect(mockRun.mock.calls[0][2].input).toBe('Note: bring a sleeping bag\n\nEvent: Youth Camp, Friday 14 March 2027, 18:00, at Camp Wartburg');
expect(mockRun.mock.calls[0][2].system).toContain('WhatsApp');
expect(mockRun.mock.calls[0][2].system).toContain('{{event.title}}');
});
test('draft a message without an event: no event placeholders offered', async () => {
mockRun.mockResolvedValue({ output: { subject: 'S', body: 'Hi {{name}}' }, status });
await draftMessage('u1', { channel: 'email', note: 'the breakfast moves to the hotel' }, now);
const system = mockRun.mock.calls[0][2].system;
expect(system).toContain('Use no other {{…}} placeholders');
expect(system).not.toContain('{{event.title}}');
});
test('improve text: the mode and where it goes shape the instructions', async () => {
+86 -1
View File
@@ -5,11 +5,17 @@ import request from 'supertest';
// way everywhere, and produce the OpenAPI document.
const mockDb = {
event: { findMany: jest.fn(), count: jest.fn(), findFirst: jest.fn() },
event: { findMany: jest.fn(), count: jest.fn(), findFirst: jest.fn(), findUnique: jest.fn() },
user: { findUnique: jest.fn() },
registration: { findMany: jest.fn(), count: jest.fn(), findUnique: jest.fn(), findUniqueOrThrow: jest.fn() },
feedbackForm: { findUnique: jest.fn() },
};
jest.mock('../src/config/db', () => mockDb);
// AI helpers: everything real except the call to the Hub.
const mockRunHelper = jest.fn();
jest.mock('../src/ai/index', () => ({ ...jest.requireActual('../src/ai/index'), runHelper: (...a: unknown[]) => mockRunHelper(...a) }));
const mockAccessLog = jest.fn(async () => undefined);
jest.mock('../src/utils/accessLog', () => ({ logDataAccess: (...a: unknown[]) => mockAccessLog(...(a as [])) }));
jest.mock('../src/utils/campuses', () => ({
campusesEnabled: jest.fn().mockResolvedValue(false),
resolveCampusFilter: jest.fn(async (v: string) => v),
@@ -693,3 +699,82 @@ describe('money (v1 handlers behind v2 routes)', () => {
expect(res.body.data).toEqual({ params: { lineId: 'l9' }, body: { resolution: 'paid' } });
});
});
describe('AI helpers (2.0)', () => {
const st = { enabled: true, capReached: false, available: true, capCents: 5000, usedCents: 1, usedPercent: 0, renewsOn: '2026-12-01', helpers: { draft_event: true, summarise_report: true, draft_message: true, summarise_feedback: true, improve_text: true } };
test('supervisors only', async () => {
signedIn = { id: 's1', role: 'staff' };
expect((await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there' })).status).toBe(403);
expect(mockRunHelper).not.toHaveBeenCalled();
});
test('improve text: the answer and the new status', async () => {
signedIn = { id: 's1', role: 'supervisor' };
mockRunHelper.mockResolvedValue({ output: { text: 'Hello there.', changes: ['Capital letter'] }, status: st });
const res = await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there', mode: 'fix', where: 'banner' });
expect(res.status).toBe(200);
expect(res.body.data).toEqual({ output: { text: 'Hello there.', changes: ['Capital letter'] }, status: st });
expect(mockRunHelper.mock.calls[0][0]).toBe('improve_text');
});
test('summarise a report: reports with people in them are refused by validation', async () => {
signedIn = { id: 's1', role: 'supervisor' };
const res = await request(app).post('/api/v2/ai/summarise-report').send({ reportKey: 'payments', title: 'Payments', columns: ['User'], rows: [['Jane']] });
expect(res.status).toBe(400);
expect(mockRunHelper).not.toHaveBeenCalled();
});
test('draft an event: the draft with warnings worked out in code', async () => {
signedIn = { id: 's1', role: 'supervisor' };
const draft = {
title: 'Camp', summary: '', description: '', startDate: '2027-03-14', startTime: '18:00', endDate: '2027-03-16', endTime: '14:00', location: '', capacity: 60, needsParentalConsent: true, ageGroup: '',
ticketOptions: [{ name: 'Camper', priceRand: 450, description: '', earlyBirdPriceRand: 0, earlyBirdUntil: '' }], sessions: [],
costs: [{ item: 'Venue', basis: '', amountRand: 34800 }], checks: [],
};
mockRunHelper.mockResolvedValue({ output: draft, status: st });
const res = await request(app).post('/api/v2/ai/draft-event').send({ notes: 'Camp Fri 14 March, R450, 60 places' });
expect(res.status).toBe(200);
expect(res.body.data.output).toMatchObject({ totalCostRand: 34800, breakEvenPlaces: 78 });
expect(res.body.data.output.warnings.join(' ')).toMatch(/is a Sunday.*R7 800 short/);
});
test('draft a message: the event is given as facts', async () => {
signedIn = { id: 's1', role: 'supervisor' };
mockDb.event.findUnique.mockResolvedValue({ title: 'Youth Camp', startDate: new Date('2027-03-12T16:00:00Z'), location: 'Camp Wartburg' });
mockRunHelper.mockResolvedValue({ output: { subject: '', body: 'Hi {{name}}!' }, status: st });
const res = await request(app).post('/api/v2/ai/draft-message').send({ channel: 'whatsapp', note: 'bring a sleeping bag', eventId: '11111111-1111-4111-8111-111111111111' });
expect(res.status).toBe(200);
expect(res.body.data.output).toEqual({ subject: '', body: 'Hi {{name}}!' });
expect(mockRunHelper.mock.calls[0][2].input).toContain('Event: Youth Camp');
});
test('summarise feedback: answers without who gave them, and an access-log entry', async () => {
signedIn = { id: 's1', role: 'supervisor' };
mockDb.feedbackForm.findUnique.mockResolvedValue({
event: { title: 'Conference' }, _count: { invites: 10 },
fields: [{ id: 'f1', type: 'rating', label: 'Overall' }, { id: 'f2', type: 'textarea', label: 'Comments' }],
responses: [
{ userId: 'u1', user: { name: 'Jane Doe' }, answers: [{ fieldId: 'f1', value: '5' }, { fieldId: 'f2', value: 'Loved it, call me on 082 555 1234' }] },
{ userId: 'u2', user: { name: 'John Roe' }, answers: [{ fieldId: 'f1', value: '3' }] },
],
});
mockRunHelper.mockResolvedValue({ output: { overall: 'Good.', liked: [], improve: [], quotes: [], nextTime: [] }, status: st });
const res = await request(app).post('/api/v2/ai/summarise-feedback').send({ eventId: '11111111-1111-4111-8111-111111111111' });
expect(res.status).toBe(200);
expect(res.body.data.output.responses).toBe(2);
const sent = mockRunHelper.mock.calls[0][2].input;
expect(sent).toContain('Overall — average 4');
expect(sent).not.toMatch(/Jane|John|082 555 1234/);
expect(mockAccessLog).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: 'ai_feedback', records: 2 }));
});
test('Hub errors keep their reason', async () => {
signedIn = { id: 's1', role: 'supervisor' };
const { ApiError } = jest.requireActual('../src/v2/errors');
mockRunHelper.mockRejectedValue(Object.assign(new ApiError('FORBIDDEN', 'Used up.'), { reason: 'AI_CAP_REACHED' }));
const res = await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there' });
expect(res.status).toBe(403);
expect(res.body.error).toMatchObject({ code: 'FORBIDDEN', reason: 'AI_CAP_REACHED' });
});
});