diff --git a/apps/events/backend/scripts/ai-eval.ts b/apps/events/backend/scripts/ai-eval.ts index c1b3549..42b8bc1 100644 --- a/apps/events/backend/scripts/ai-eval.ts +++ b/apps/events/backend/scripts/ai-eval.ts @@ -10,6 +10,7 @@ * The examples come from the October 2026 model trial; the inputs are made up and hold no * personal information. */ +import 'dotenv/config'; import { prisma } from '../src/v2/legacy'; import { draftEvent, summariseReport, draftMessage, summariseFeedback, improveText } from '../src/ai/helpers'; @@ -83,7 +84,8 @@ const CASES: Case[] = [ { label: 'Food', type: 'rating', answered: 108, average: 3.6 }, { label: 'Comments', type: 'text', answered: FEEDBACK_COMMENTS.length, texts: FEEDBACK_COMMENTS }, ], }); - const count = (re: RegExp) => summary.improve.find(t => re.test(t.theme))?.mentions ?? -1; + // A theme may be split (e.g. "lunch queues" and "vegetarian food"): add up every match. + const count = (re: RegExp) => summary.improve.filter(t => re.test(t.theme)).reduce((n, t) => n + t.mentions, 0); return { output: summary, checks: [ @@ -104,6 +106,7 @@ const CASES: Case[] = [ ['uses {{name}}', message.body.includes('{{name}}')], ['keeps 7:30, sleeping bag, car park', /7:30/.test(message.body) && /sleeping bag/i.test(message.body) && /car park/i.test(message.body)], ['no "reminda" (seen with Haiku)', !/reminda/i.test(message.body)], + ['no event placeholders without an event', !/\{\{event\./.test(message.body)], ], }; }, @@ -118,6 +121,7 @@ const CASES: Case[] = [ ['has a subject', message.subject.trim().length > 5], ['keeps the facts', /Protea/.test(message.body) && /7:00/.test(message.body) && /R120/.test(message.body)], ['the email address came back intact', message.body.includes('events@church.example.org')], + ['no event placeholders without an event', !/\{\{event\./.test(`${message.subject} ${message.body}`)], ], }; }, diff --git a/apps/events/backend/src/ai/helpers.ts b/apps/events/backend/src/ai/helpers.ts index efe49f8..319ae96 100644 --- a/apps/events/backend/src/ai/helpers.ts +++ b/apps/events/backend/src/ai/helpers.ts @@ -216,13 +216,15 @@ export type MessageAnswer = { subject: string; body: string }; const MESSAGE_SCHEMA = obj({ subject: str, body: str }, { subject: 'Empty for WhatsApp' }); -const MESSAGE_SYSTEM = (channel: 'email' | 'whatsapp', now: Date) => [ +const MESSAGE_SYSTEM = (channel: 'email' | 'whatsapp', now: Date, withEvent: boolean) => [ 'You write messages from a church events team to people registered for an event, from the organiser\'s short note.', todayLine(now), channel === 'whatsapp' ? '- WhatsApp: plain text, at most 3 short sentences, at most one emoji. "subject" is empty.' : '- Email: a clear subject, then the body in short plain-text paragraphs, the most important thing first. Start with "Hi {{name}}," and end with "The events team".', - '- Use {{name}} for the person\'s first name. {{event.title}}, {{event.start}} and {{event.link}} are filled in for each event, if useful.', + withEvent + ? '- Use {{name}} for the person\'s first name. {{event.title}}, {{event.start}} and {{event.link}} are filled in with the event\'s details; use them only where the message needs them, and don\'t add a separate block of event details.' + : '- Use {{name}} for the person\'s first name. Use no other {{…}} placeholders: name the event as the note does.', '- Use only the facts in the note and the event details; never invent times, places, prices, links or contact details, or add advice the note doesn\'t give.', '- Spell carefully: this goes to everyone registered.', KEEP_PLACEHOLDERS, @@ -234,7 +236,7 @@ export async function draftMessage(userId: string, input: { channel: 'email' | ' const p = protect(input.note); const facts = input.event ? `\n\nEvent: ${input.event.title}, ${input.event.start}${input.event.location ? `, at ${input.event.location}` : ''}` : ''; const { output, status } = await runHelper('draft_message', userId, { - system: MESSAGE_SYSTEM(input.channel, now), input: `Note: ${p.text}${facts}`, schema: MESSAGE_SCHEMA, maxTokens: input.channel === 'whatsapp' ? 600 : 1500, effort: 'low', + system: MESSAGE_SYSTEM(input.channel, now, !!input.event), input: `Note: ${p.text}${facts}`, schema: MESSAGE_SCHEMA, maxTokens: input.channel === 'whatsapp' ? 600 : 1500, effort: 'low', }); const message = restore(output as MessageAnswer, p.found); return { message: input.channel === 'whatsapp' ? { subject: '', body: message.body } : message, status }; diff --git a/apps/events/backend/tests/aiHelpers.test.ts b/apps/events/backend/tests/aiHelpers.test.ts index 74bd40e..0103352 100644 --- a/apps/events/backend/tests/aiHelpers.test.ts +++ b/apps/events/backend/tests/aiHelpers.test.ts @@ -110,6 +110,15 @@ describe('what the helpers send', () => { expect(r.message).toEqual({ subject: '', body: 'Hi {{name}}!' }); expect(mockRun.mock.calls[0][2].input).toBe('Note: bring a sleeping bag\n\nEvent: Youth Camp, Friday 14 March 2027, 18:00, at Camp Wartburg'); expect(mockRun.mock.calls[0][2].system).toContain('WhatsApp'); + expect(mockRun.mock.calls[0][2].system).toContain('{{event.title}}'); + }); + + test('draft a message without an event: no event placeholders offered', async () => { + mockRun.mockResolvedValue({ output: { subject: 'S', body: 'Hi {{name}}' }, status }); + await draftMessage('u1', { channel: 'email', note: 'the breakfast moves to the hotel' }, now); + const system = mockRun.mock.calls[0][2].system; + expect(system).toContain('Use no other {{…}} placeholders'); + expect(system).not.toContain('{{event.title}}'); }); test('improve text: the mode and where it goes shape the instructions', async () => { diff --git a/apps/events/backend/tests/v2.test.ts b/apps/events/backend/tests/v2.test.ts index 58c3a62..84a1dc6 100644 --- a/apps/events/backend/tests/v2.test.ts +++ b/apps/events/backend/tests/v2.test.ts @@ -5,11 +5,17 @@ import request from 'supertest'; // way everywhere, and produce the OpenAPI document. const mockDb = { - event: { findMany: jest.fn(), count: jest.fn(), findFirst: jest.fn() }, + event: { findMany: jest.fn(), count: jest.fn(), findFirst: jest.fn(), findUnique: jest.fn() }, user: { findUnique: jest.fn() }, registration: { findMany: jest.fn(), count: jest.fn(), findUnique: jest.fn(), findUniqueOrThrow: jest.fn() }, + feedbackForm: { findUnique: jest.fn() }, }; jest.mock('../src/config/db', () => mockDb); +// AI helpers: everything real except the call to the Hub. +const mockRunHelper = jest.fn(); +jest.mock('../src/ai/index', () => ({ ...jest.requireActual('../src/ai/index'), runHelper: (...a: unknown[]) => mockRunHelper(...a) })); +const mockAccessLog = jest.fn(async () => undefined); +jest.mock('../src/utils/accessLog', () => ({ logDataAccess: (...a: unknown[]) => mockAccessLog(...(a as [])) })); jest.mock('../src/utils/campuses', () => ({ campusesEnabled: jest.fn().mockResolvedValue(false), resolveCampusFilter: jest.fn(async (v: string) => v), @@ -693,3 +699,82 @@ describe('money (v1 handlers behind v2 routes)', () => { expect(res.body.data).toEqual({ params: { lineId: 'l9' }, body: { resolution: 'paid' } }); }); }); + +describe('AI helpers (2.0)', () => { + const st = { enabled: true, capReached: false, available: true, capCents: 5000, usedCents: 1, usedPercent: 0, renewsOn: '2026-12-01', helpers: { draft_event: true, summarise_report: true, draft_message: true, summarise_feedback: true, improve_text: true } }; + + test('supervisors only', async () => { + signedIn = { id: 's1', role: 'staff' }; + expect((await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there' })).status).toBe(403); + expect(mockRunHelper).not.toHaveBeenCalled(); + }); + + test('improve text: the answer and the new status', async () => { + signedIn = { id: 's1', role: 'supervisor' }; + mockRunHelper.mockResolvedValue({ output: { text: 'Hello there.', changes: ['Capital letter'] }, status: st }); + const res = await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there', mode: 'fix', where: 'banner' }); + expect(res.status).toBe(200); + expect(res.body.data).toEqual({ output: { text: 'Hello there.', changes: ['Capital letter'] }, status: st }); + expect(mockRunHelper.mock.calls[0][0]).toBe('improve_text'); + }); + + test('summarise a report: reports with people in them are refused by validation', async () => { + signedIn = { id: 's1', role: 'supervisor' }; + const res = await request(app).post('/api/v2/ai/summarise-report').send({ reportKey: 'payments', title: 'Payments', columns: ['User'], rows: [['Jane']] }); + expect(res.status).toBe(400); + expect(mockRunHelper).not.toHaveBeenCalled(); + }); + + test('draft an event: the draft with warnings worked out in code', async () => { + signedIn = { id: 's1', role: 'supervisor' }; + const draft = { + title: 'Camp', summary: '', description: '', startDate: '2027-03-14', startTime: '18:00', endDate: '2027-03-16', endTime: '14:00', location: '', capacity: 60, needsParentalConsent: true, ageGroup: '', + ticketOptions: [{ name: 'Camper', priceRand: 450, description: '', earlyBirdPriceRand: 0, earlyBirdUntil: '' }], sessions: [], + costs: [{ item: 'Venue', basis: '', amountRand: 34800 }], checks: [], + }; + mockRunHelper.mockResolvedValue({ output: draft, status: st }); + const res = await request(app).post('/api/v2/ai/draft-event').send({ notes: 'Camp Fri 14 March, R450, 60 places' }); + expect(res.status).toBe(200); + expect(res.body.data.output).toMatchObject({ totalCostRand: 34800, breakEvenPlaces: 78 }); + expect(res.body.data.output.warnings.join(' ')).toMatch(/is a Sunday.*R7 800 short/); + }); + + test('draft a message: the event is given as facts', async () => { + signedIn = { id: 's1', role: 'supervisor' }; + mockDb.event.findUnique.mockResolvedValue({ title: 'Youth Camp', startDate: new Date('2027-03-12T16:00:00Z'), location: 'Camp Wartburg' }); + mockRunHelper.mockResolvedValue({ output: { subject: '', body: 'Hi {{name}}!' }, status: st }); + const res = await request(app).post('/api/v2/ai/draft-message').send({ channel: 'whatsapp', note: 'bring a sleeping bag', eventId: '11111111-1111-4111-8111-111111111111' }); + expect(res.status).toBe(200); + expect(res.body.data.output).toEqual({ subject: '', body: 'Hi {{name}}!' }); + expect(mockRunHelper.mock.calls[0][2].input).toContain('Event: Youth Camp'); + }); + + test('summarise feedback: answers without who gave them, and an access-log entry', async () => { + signedIn = { id: 's1', role: 'supervisor' }; + mockDb.feedbackForm.findUnique.mockResolvedValue({ + event: { title: 'Conference' }, _count: { invites: 10 }, + fields: [{ id: 'f1', type: 'rating', label: 'Overall' }, { id: 'f2', type: 'textarea', label: 'Comments' }], + responses: [ + { userId: 'u1', user: { name: 'Jane Doe' }, answers: [{ fieldId: 'f1', value: '5' }, { fieldId: 'f2', value: 'Loved it, call me on 082 555 1234' }] }, + { userId: 'u2', user: { name: 'John Roe' }, answers: [{ fieldId: 'f1', value: '3' }] }, + ], + }); + mockRunHelper.mockResolvedValue({ output: { overall: 'Good.', liked: [], improve: [], quotes: [], nextTime: [] }, status: st }); + const res = await request(app).post('/api/v2/ai/summarise-feedback').send({ eventId: '11111111-1111-4111-8111-111111111111' }); + expect(res.status).toBe(200); + expect(res.body.data.output.responses).toBe(2); + const sent = mockRunHelper.mock.calls[0][2].input; + expect(sent).toContain('Overall — average 4'); + expect(sent).not.toMatch(/Jane|John|082 555 1234/); + expect(mockAccessLog).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: 'ai_feedback', records: 2 })); + }); + + test('Hub errors keep their reason', async () => { + signedIn = { id: 's1', role: 'supervisor' }; + const { ApiError } = jest.requireActual('../src/v2/errors'); + mockRunHelper.mockRejectedValue(Object.assign(new ApiError('FORBIDDEN', 'Used up.'), { reason: 'AI_CAP_REACHED' })); + const res = await request(app).post('/api/v2/ai/improve-text').send({ text: 'hello there' }); + expect(res.status).toBe(403); + expect(res.body.error).toMatchObject({ code: 'FORBIDDEN', reason: 'AI_CAP_REACHED' }); + }); +});