WIP 2.0.0: message drafts use event placeholders only with an event
Found by the first real eval run (Sonnet 5.5 through the local Hub,
19/19 checks): without an event the drafts still used {{event.title}}
and added an event-details block. The instructions now offer those
placeholders only when an event is given; the eval checks for it, and
adds up split themes when counting feedback. Route tests for the five
AI helpers in v2.test.ts; the eval script loads backend/.env.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ls1vMeckMWrKPfc1MMyMJF
This commit is contained in:
@@ -10,6 +10,7 @@
|
||||
* The examples come from the October 2026 model trial; the inputs are made up and hold no
|
||||
* personal information.
|
||||
*/
|
||||
import 'dotenv/config';
|
||||
import { prisma } from '../src/v2/legacy';
|
||||
import { draftEvent, summariseReport, draftMessage, summariseFeedback, improveText } from '../src/ai/helpers';
|
||||
|
||||
@@ -83,7 +84,8 @@ const CASES: Case[] = [
|
||||
{ label: 'Food', type: 'rating', answered: 108, average: 3.6 }, { label: 'Comments', type: 'text', answered: FEEDBACK_COMMENTS.length, texts: FEEDBACK_COMMENTS },
|
||||
],
|
||||
});
|
||||
const count = (re: RegExp) => summary.improve.find(t => re.test(t.theme))?.mentions ?? -1;
|
||||
// A theme may be split (e.g. "lunch queues" and "vegetarian food"): add up every match.
|
||||
const count = (re: RegExp) => summary.improve.filter(t => re.test(t.theme)).reduce((n, t) => n + t.mentions, 0);
|
||||
return {
|
||||
output: summary,
|
||||
checks: [
|
||||
@@ -104,6 +106,7 @@ const CASES: Case[] = [
|
||||
['uses {{name}}', message.body.includes('{{name}}')],
|
||||
['keeps 7:30, sleeping bag, car park', /7:30/.test(message.body) && /sleeping bag/i.test(message.body) && /car park/i.test(message.body)],
|
||||
['no "reminda" (seen with Haiku)', !/reminda/i.test(message.body)],
|
||||
['no event placeholders without an event', !/\{\{event\./.test(message.body)],
|
||||
],
|
||||
};
|
||||
},
|
||||
@@ -118,6 +121,7 @@ const CASES: Case[] = [
|
||||
['has a subject', message.subject.trim().length > 5],
|
||||
['keeps the facts', /Protea/.test(message.body) && /7:00/.test(message.body) && /R120/.test(message.body)],
|
||||
['the email address came back intact', message.body.includes('events@church.example.org')],
|
||||
['no event placeholders without an event', !/\{\{event\./.test(`${message.subject} ${message.body}`)],
|
||||
],
|
||||
};
|
||||
},
|
||||
|
||||
@@ -216,13 +216,15 @@ export type MessageAnswer = { subject: string; body: string };
|
||||
|
||||
const MESSAGE_SCHEMA = obj({ subject: str, body: str }, { subject: 'Empty for WhatsApp' });
|
||||
|
||||
const MESSAGE_SYSTEM = (channel: 'email' | 'whatsapp', now: Date) => [
|
||||
const MESSAGE_SYSTEM = (channel: 'email' | 'whatsapp', now: Date, withEvent: boolean) => [
|
||||
'You write messages from a church events team to people registered for an event, from the organiser\'s short note.',
|
||||
todayLine(now),
|
||||
channel === 'whatsapp'
|
||||
? '- WhatsApp: plain text, at most 3 short sentences, at most one emoji. "subject" is empty.'
|
||||
: '- Email: a clear subject, then the body in short plain-text paragraphs, the most important thing first. Start with "Hi {{name}}," and end with "The events team".',
|
||||
'- Use {{name}} for the person\'s first name. {{event.title}}, {{event.start}} and {{event.link}} are filled in for each event, if useful.',
|
||||
withEvent
|
||||
? '- Use {{name}} for the person\'s first name. {{event.title}}, {{event.start}} and {{event.link}} are filled in with the event\'s details; use them only where the message needs them, and don\'t add a separate block of event details.'
|
||||
: '- Use {{name}} for the person\'s first name. Use no other {{…}} placeholders: name the event as the note does.',
|
||||
'- Use only the facts in the note and the event details; never invent times, places, prices, links or contact details, or add advice the note doesn\'t give.',
|
||||
'- Spell carefully: this goes to everyone registered.',
|
||||
KEEP_PLACEHOLDERS,
|
||||
@@ -234,7 +236,7 @@ export async function draftMessage(userId: string, input: { channel: 'email' | '
|
||||
const p = protect(input.note);
|
||||
const facts = input.event ? `\n\nEvent: ${input.event.title}, ${input.event.start}${input.event.location ? `, at ${input.event.location}` : ''}` : '';
|
||||
const { output, status } = await runHelper('draft_message', userId, {
|
||||
system: MESSAGE_SYSTEM(input.channel, now), input: `Note: ${p.text}${facts}`, schema: MESSAGE_SCHEMA, maxTokens: input.channel === 'whatsapp' ? 600 : 1500, effort: 'low',
|
||||
system: MESSAGE_SYSTEM(input.channel, now, !!input.event), input: `Note: ${p.text}${facts}`, schema: MESSAGE_SCHEMA, maxTokens: input.channel === 'whatsapp' ? 600 : 1500, effort: 'low',
|
||||
});
|
||||
const message = restore(output as MessageAnswer, p.found);
|
||||
return { message: input.channel === 'whatsapp' ? { subject: '', body: message.body } : message, status };
|
||||
|
||||
@@ -110,6 +110,15 @@ describe('what the helpers send', () => {
|
||||
expect(r.message).toEqual({ subject: '', body: 'Hi {{name}}!' });
|
||||
expect(mockRun.mock.calls[0][2].input).toBe('Note: bring a sleeping bag\n\nEvent: Youth Camp, Friday 14 March 2027, 18:00, at Camp Wartburg');
|
||||
expect(mockRun.mock.calls[0][2].system).toContain('WhatsApp');
|
||||
expect(mockRun.mock.calls[0][2].system).toContain('{{event.title}}');
|
||||
});
|
||||
|
||||
test('draft a message without an event: no event placeholders offered', async () => {
|
||||
mockRun.mockResolvedValue({ output: { subject: 'S', body: 'Hi {{name}}' }, status });
|
||||
await draftMessage('u1', { channel: 'email', note: 'the breakfast moves to the hotel' }, now);
|
||||
const system = mockRun.mock.calls[0][2].system;
|
||||
expect(system).toContain('Use no other {{…}} placeholders');
|
||||
expect(system).not.toContain('{{event.title}}');
|
||||
});
|
||||
|
||||
test('improve text: the mode and where it goes shape the instructions', async () => {
|
||||
|
||||
@@ -5,11 +5,17 @@ import request from 'supertest';
|
||||
// way everywhere, and produce the OpenAPI document.
|
||||
|
||||
const mockDb = {
|
||||
event: { findMany: jest.fn(), count: jest.fn(), findFirst: jest.fn() },
|
||||
event: { findMany: jest.fn(), count: jest.fn(), findFirst: jest.fn(), findUnique: jest.fn() },
|
||||
user: { findUnique: jest.fn() },
|
||||
registration: { findMany: jest.fn(), count: jest.fn(), findUnique: jest.fn(), findUniqueOrThrow: jest.fn() },
|
||||
feedbackForm: { findUnique: jest.fn() },
|
||||
};
|
||||
jest.mock('../src/config/db', () => mockDb);
|
||||
// AI helpers: everything real except the call to the Hub.
|
||||
const mockRunHelper = jest.fn();
|
||||
jest.mock('../src/ai/index', () => ({ ...jest.requireActual('../src/ai/index'), runHelper: (...a: unknown[]) => mockRunHelper(...a) }));
|
||||
const mockAccessLog = jest.fn(async () => undefined);
|
||||
jest.mock('../src/utils/accessLog', () => ({ logDataAccess: (...a: unknown[]) => mockAccessLog(...(a as [])) }));
|
||||
jest.mock('../src/utils/campuses', () => ({
|
||||
campusesEnabled: jest.fn().mockResolvedValue(false),
|
||||
resolveCampusFilter: jest.fn(async (v: string) => v),
|
||||
@@ -693,3 +699,82 @@ describe('money (v1 handlers behind v2 routes)', () => {
|
||||
expect(res.body.data).toEqual({ params: { lineId: 'l9' }, body: { resolution: 'paid' } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('AI helpers (2.0)', () => {
|
||||
const st = { enabled: true, capReached: false, available: true, capCents: 5000, usedCents: 1, usedPercent: 0, renewsOn: '2026-12-01', helpers: { draft_event: true, summarise_report: true, draft_message: true, summarise_feedback: true, improve_text: true } };
|
||||
|
||||
test('supervisors only', async () => {
|
||||
signedIn = { id: 's1', role: 'staff' };
|
||||
expect((await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there' })).status).toBe(403);
|
||||
expect(mockRunHelper).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('improve text: the answer and the new status', async () => {
|
||||
signedIn = { id: 's1', role: 'supervisor' };
|
||||
mockRunHelper.mockResolvedValue({ output: { text: 'Hello there.', changes: ['Capital letter'] }, status: st });
|
||||
const res = await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there', mode: 'fix', where: 'banner' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.data).toEqual({ output: { text: 'Hello there.', changes: ['Capital letter'] }, status: st });
|
||||
expect(mockRunHelper.mock.calls[0][0]).toBe('improve_text');
|
||||
});
|
||||
|
||||
test('summarise a report: reports with people in them are refused by validation', async () => {
|
||||
signedIn = { id: 's1', role: 'supervisor' };
|
||||
const res = await request(app).post('/api/v2/ai/summarise-report').send({ reportKey: 'payments', title: 'Payments', columns: ['User'], rows: [['Jane']] });
|
||||
expect(res.status).toBe(400);
|
||||
expect(mockRunHelper).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('draft an event: the draft with warnings worked out in code', async () => {
|
||||
signedIn = { id: 's1', role: 'supervisor' };
|
||||
const draft = {
|
||||
title: 'Camp', summary: '', description: '', startDate: '2027-03-14', startTime: '18:00', endDate: '2027-03-16', endTime: '14:00', location: '', capacity: 60, needsParentalConsent: true, ageGroup: '',
|
||||
ticketOptions: [{ name: 'Camper', priceRand: 450, description: '', earlyBirdPriceRand: 0, earlyBirdUntil: '' }], sessions: [],
|
||||
costs: [{ item: 'Venue', basis: '', amountRand: 34800 }], checks: [],
|
||||
};
|
||||
mockRunHelper.mockResolvedValue({ output: draft, status: st });
|
||||
const res = await request(app).post('/api/v2/ai/draft-event').send({ notes: 'Camp Fri 14 March, R450, 60 places' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.data.output).toMatchObject({ totalCostRand: 34800, breakEvenPlaces: 78 });
|
||||
expect(res.body.data.output.warnings.join(' ')).toMatch(/is a Sunday.*R7 800 short/);
|
||||
});
|
||||
|
||||
test('draft a message: the event is given as facts', async () => {
|
||||
signedIn = { id: 's1', role: 'supervisor' };
|
||||
mockDb.event.findUnique.mockResolvedValue({ title: 'Youth Camp', startDate: new Date('2027-03-12T16:00:00Z'), location: 'Camp Wartburg' });
|
||||
mockRunHelper.mockResolvedValue({ output: { subject: '', body: 'Hi {{name}}!' }, status: st });
|
||||
const res = await request(app).post('/api/v2/ai/draft-message').send({ channel: 'whatsapp', note: 'bring a sleeping bag', eventId: '11111111-1111-4111-8111-111111111111' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.data.output).toEqual({ subject: '', body: 'Hi {{name}}!' });
|
||||
expect(mockRunHelper.mock.calls[0][2].input).toContain('Event: Youth Camp');
|
||||
});
|
||||
|
||||
test('summarise feedback: answers without who gave them, and an access-log entry', async () => {
|
||||
signedIn = { id: 's1', role: 'supervisor' };
|
||||
mockDb.feedbackForm.findUnique.mockResolvedValue({
|
||||
event: { title: 'Conference' }, _count: { invites: 10 },
|
||||
fields: [{ id: 'f1', type: 'rating', label: 'Overall' }, { id: 'f2', type: 'textarea', label: 'Comments' }],
|
||||
responses: [
|
||||
{ userId: 'u1', user: { name: 'Jane Doe' }, answers: [{ fieldId: 'f1', value: '5' }, { fieldId: 'f2', value: 'Loved it, call me on 082 555 1234' }] },
|
||||
{ userId: 'u2', user: { name: 'John Roe' }, answers: [{ fieldId: 'f1', value: '3' }] },
|
||||
],
|
||||
});
|
||||
mockRunHelper.mockResolvedValue({ output: { overall: 'Good.', liked: [], improve: [], quotes: [], nextTime: [] }, status: st });
|
||||
const res = await request(app).post('/api/v2/ai/summarise-feedback').send({ eventId: '11111111-1111-4111-8111-111111111111' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.data.output.responses).toBe(2);
|
||||
const sent = mockRunHelper.mock.calls[0][2].input;
|
||||
expect(sent).toContain('Overall — average 4');
|
||||
expect(sent).not.toMatch(/Jane|John|082 555 1234/);
|
||||
expect(mockAccessLog).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: 'ai_feedback', records: 2 }));
|
||||
});
|
||||
|
||||
test('Hub errors keep their reason', async () => {
|
||||
signedIn = { id: 's1', role: 'supervisor' };
|
||||
const { ApiError } = jest.requireActual('../src/v2/errors');
|
||||
mockRunHelper.mockRejectedValue(Object.assign(new ApiError('FORBIDDEN', 'Used up.'), { reason: 'AI_CAP_REACHED' }));
|
||||
const res = await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there' });
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.error).toMatchObject({ code: 'FORBIDDEN', reason: 'AI_CAP_REACHED' });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user