hope-events/backend/scripts/ai-eval.ts

/**
 * The AI helpers' eval set (roadmap §15): `npm run ai:eval` in backend/.
 *
 * Runs fixed examples of all five helpers through this site's real Hub connection (so the site must
 * be connected and its licence must have an AI allowance), checks the facts that matter
 * automatically, and prints every answer for a person to read. Run it after changing a helper's
 * instructions or the model on the Hub, and compare with the last run. It costs about R1 a run
 * with Claude Sonnet 5.5; the calls count against this site's allowance.
 *
 * The examples come from the October 2026 model trial; the inputs are made up and hold no
 * personal information.
 */
import 'dotenv/config';
import { prisma } from '../src/v2/legacy';
import { draftEvent, summariseReport, draftMessage, summariseFeedback, improveText } from '../src/ai/helpers';

type Check = [label: string, ok: boolean];
type Case = { name: string; run: (userId: string) => Promise<{ output: unknown; checks: Check[] }> };

const has = (text: unknown, re: RegExp) => re.test(JSON.stringify(text));
const near = (n: number, target: number, give = 1) => Math.abs(n - target) <= give;

const CAMP_NOTES = 'Youth camp for grades 8–12, Fri 14 March 18:00 to Sun 16 March 14:00 at Camp Wartburg near Pietermaritzburg. 60 places. R450 per person, R400 if they book by 31 January. Parents must give consent. Theme "Rooted" (Colossians 2:7). Costs: venue R180 per person per night including meals, bus hire R6 500 return, speaker gift R1 000, T-shirts R95 each. Friday evening worship, Saturday sessions morning and evening with sport in the afternoon, Sunday morning service before we leave. Bring a sleeping bag, Bible and a torch.';

const FEEDBACK_COMMENTS = [
  'The keynote on Saturday morning was exactly what I needed this year.', 'Could not hear a thing in Hall B, the sound kept cutting out.',
  'Lunch queue took 40 minutes, we missed the start of the afternoon session.', 'Loved the worship, so good to be together again.',
  'Breakouts were too short, just as it got deep we had to leave.', 'Check-in with the QR code was so quick, well done.',
  'Parking was a nightmare, we parked on the road and walked.', 'Please have childcare next year, I almost could not come.',
  'Speaker in the parenting breakout was brilliant.', 'Sound in Hall B again, the mic was crackling the whole time.',
  'Food ran out of vegetarian options by the time we got there.', 'The whole day felt so well organised.',
  'More time for breakouts please, and maybe fewer of them.', 'Lunch took forever.', 'Loved the gift bags!',
  'Hall A was freezing, the aircon was too cold.', 'The app reminders the day before were really helpful.',
  'Would love a longer Q&A with the main speaker.', 'Queue for lunch was very long, maybe stagger the breakouts?',
  'Parking marshals would help a lot.', 'Worship team was amazing.', 'Childcare or a kids programme would let young moms come.',
  'Some of the slides were hard to read from the back.', 'Best conference yet, thank you to the whole team.',
  'I would pay a bit more for a better lunch.', 'Hall B sound please fix it.',
];

const CASES: Case[] = [
  {
    name: 'Draft an event — full notes with costs',
    async run(userId) {
      const r = await draftEvent(userId, CAMP_NOTES);
      const d = r.draft;
      return {
        output: r,
        checks: [
          ['price R450, early bird R400 until 31 January', d.ticketOptions[0]?.priceRand === 450 && d.ticketOptions[0]?.earlyBirdPriceRand === 400 && d.ticketOptions[0]?.earlyBirdUntil.endsWith('-01-31')],
          ['60 places, parental consent', d.capacity === 60 && d.needsParentalConsent],
          ['budget R34 800, break-even 78', r.totalCostRand === 34800 && r.breakEvenPlaces === 78],
          ['warns the camp is short even when full', r.warnings.some(w => w.includes('short'))],
          ['times not in the notes left empty (only 18:00 and 14:00 given)', d.sessions.every(s => !s.startTime || s.startTime === '18:00')],
        ],
      };
    },
  },
  {
    name: 'Summarise a report — a month of payments',
    async run(userId) {
      const { summary } = await summariseReport(userId, {
        reportKey: 'revenue', title: 'Revenue summary — October 2026 (September for comparison)',
        columns: ['Method', 'October payments', 'October amount', 'September payments', 'September amount'],
        rows: [['Card', 96, 'R42 300', 81, 'R35 100'], ['EFT', 14, 'R18 000', 17, 'R21 500'], ['Cash at the door', 31, 'R6 250', 22, 'R4 100'], ['Refunds', 4, 'R2 700', 2, 'R450']],
        stats: [{ label: 'Card payments waiting for the provider over 2 days', value: '3 (R1 350)' }],
      });
      return {
        output: summary,
        checks: [
          ['October total R66 550', has(summary, /66[\s ]?550/)],
          ['up on September (R60 700)', has(summary, /60[\s ]?700|5[\s ]?850|9[.,]6/)],
          ['the 3 waiting card payments need checking', has(summary.watchOut, /1[\s ]?350|3 (online |card )?payments/i)],
        ],
      };
    },
  },
  {
    name: 'Summarise feedback — women\'s conference',
    async run(userId) {
      const { summary } = await summariseFeedback(userId, {
        event: { title: 'Women\'s Conference 2026' }, responses: 112, invited: 186,
        questions: [
          { label: 'Overall', type: 'rating', answered: 112, average: 4.4 }, { label: 'Venue', type: 'rating', answered: 110, average: 3.9 },
          { label: 'Food', type: 'rating', answered: 108, average: 3.6 }, { label: 'Comments', type: 'text', answered: FEEDBACK_COMMENTS.length, texts: FEEDBACK_COMMENTS },
        ],
      });
      // A theme may be split (e.g. "lunch queues" and "vegetarian food"): add up every match.
      const count = (re: RegExp) => summary.improve.filter(t => re.test(t.theme)).reduce((n, t) => n + t.mentions, 0);
      return {
        output: summary,
        checks: [
          ['lunch/food raised by about 5', near(count(/lunch|food|queue/i), 5)],
          ['Hall B sound raised by 3', near(count(/sound|hall b/i), 3, 0)],
          ['quotes copied exactly', summary.quotes.every(q => FEEDBACK_COMMENTS.includes(q))],
        ],
      };
    },
  },
  {
    name: 'Draft a message — WhatsApp reminder',
    async run(userId) {
      const { message } = await draftMessage(userId, { channel: 'whatsapp', note: 'Youth camp this weekend: remind them to bring a sleeping bag; the bus leaves at 7:30 on Friday from the church car park.' });
      return {
        output: message,
        checks: [
          ['uses {{name}}', message.body.includes('{{name}}')],
          ['keeps 7:30, sleeping bag, car park', /7:30/.test(message.body) && /sleeping bag/i.test(message.body) && /car park/i.test(message.body)],
          ['no "reminda" (seen with Haiku)', !/reminda/i.test(message.body)],
          ['no event placeholders without an event', !/\{\{event\./.test(message.body)],
        ],
      };
    },
  },
  {
    name: 'Draft a message — email about a venue change',
    async run(userId) {
      const { message } = await draftMessage(userId, { channel: 'email', note: 'Men\'s breakfast this Saturday is moving from the church hall to the Protea Hotel on Main Road because the hall is being renovated. Same time, 7:00. Park at the back of the hotel. Price stays R120, nothing to do if you\'ve paid. Speaker is still Pastor Mike. Questions: events@church.example.org' });
      return {
        output: message,
        checks: [
          ['has a subject', message.subject.trim().length > 5],
          ['keeps the facts', /Protea/.test(message.body) && /7:00/.test(message.body) && /R120/.test(message.body)],
          ['the email address came back intact', message.body.includes('events@church.example.org')],
          ['no event placeholders without an event', !/\{\{event\./.test(`${message.subject} ${message.body}`)],
        ],
      };
    },
  },
  {
    name: 'Improve this text — a messy description',
    async run(userId) {
      const { result } = await improveText(userId, { text: 'join us for are anual family fun day!!! theres jumping castles, food stalls and a talent show for the kids. Bring the hole family its going to be awesome. tickets R50 for adults kids under 12 free. starts 9 till 2 on the church field, if it rains we move inside to the hall. please register so we know how many boerewors rolls to make lol', mode: 'improve', where: 'description' });
      return {
        output: result,
        checks: [
          ['spelling fixed', /annual/i.test(result.text) && /whole family/i.test(result.text) && !/\bare anual\b/.test(result.text)],
          ['every fact kept (R50, under 12, 9 to 2, hall)', /R50/.test(result.text) && /12/.test(result.text) && /\b9\b/.test(result.text) && /\b2\b/.test(result.text) && /hall/i.test(result.text)],
        ],
      };
    },
  },
];

(async () => {
  const admin = await prisma.user.findFirst({ where: { role: 'admin', isActive: true }, select: { id: true } });
  if (!admin) throw new Error('No active admin to log the calls against.');
  let passed = 0;
  let total = 0;
  for (const c of CASES) {
    console.log(`\n━━ ${c.name}`);
    try {
      const { output, checks } = await c.run(admin.id);
      for (const [label, ok] of checks) {
        total++;
        if (ok) passed++;
        console.log(`  ${ok ? '✓' : '✗'} ${label}`);
      }
      console.log(JSON.stringify(output, null, 2).split('\n').map(l => `    ${l}`).join('\n'));
    } catch (e) {
      total++;
      console.log(`  ✗ failed: ${(e as Error).message}`);
    }
  }
  console.log(`\n${passed} of ${total} checks passed. Read the answers too: the checks only cover the facts.`);
  await prisma.$disconnect();
  process.exit(passed === total ? 0 : 1);
})().catch(async e => {
  console.error(e);
  await prisma.$disconnect();
  process.exit(1);
});