tested/scripts/import-index.mjs

#!/usr/bin/env node
// Converts the owner's question index (Markdown export of the "Bible Defense — Question Index" doc)
// into src/data/questions.json. Run with: npm run import
// Usage: node scripts/import-index.mjs [input.md] [output.json]

import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
import { dirname } from "node:path";

const input = process.argv[2] ?? "content/source/question-index.md";
const output = process.argv[3] ?? "src/data/questions.json";

const STATUS = { green: "green", yellow: "yellow", red: "red" };
const DOMAINS = new Set([
  "Scripture",
  "History",
  "Archaeology",
  "Science",
  "Philosophy",
  "Textual Criticism",
  "Context",
  "Theology",
]);
// Article files are named after the question id, lowercase, dots as dashes: 3B.10 -> 3b-10.mdx
const articleFile = (id) => `src/content/articles/${id.replace(/\./g, "-").toLowerCase()}.mdx`;
const NOTE_SPLIT = /\s*\b(?:My Notes|Note)\s*:\s*/i;

const decode = (s) =>
  s
    .replace(/ /g, " ")
    .replace(/&/g, "&")
    .replace(/"/g, '"')
    .replace(/'/g, "'")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/\\\n/g, " ") // hard line breaks inside cells
    .replace(/\s+/g, " ")
    .trim();

const slugify = (s) =>
  s
    .toLowerCase()
    .normalize("NFKD")
    .replace(/[\u0300-\u036f]/g, "")
    .replace(/['’"]/g, "")
    .replace(/[^a-z0-9]+/g, "-")
    .replace(/^-|-$/g, "")
    .slice(0, 70)
    .replace(/-$/, "");

// Join physical lines so each table row is one logical line (rows can span lines via "\" breaks).
function logicalLines(md) {
  const raw = md.replace(/\r\n/g, "\n").split("\n");
  const out = [];
  for (const line of raw) {
    const prev = out[out.length - 1];
    if (prev !== undefined && prev.startsWith("|") && !prev.trimEnd().endsWith("|")) {
      out[out.length - 1] = prev + "\n" + line;
    } else {
      out.push(line);
    }
  }
  return out;
}

const splitRow = (line) =>
  line
    .trim()
    .replace(/^\|/, "")
    .replace(/\|$/, "")
    .split(/(?<!\\)\|/)
    .map((c) => c.trim());

// "3B.1–3B.9 Apocrypha books" -> ["3B.1", ..., "3B.9"]; "3B Missing books" -> ["group:3B"]; "4.1 Text changed?" -> ["4.1"]
function parseRelatedLabel(label, problems, ctx) {
  const range = label.match(/^(\d+B?)\.(\d+)\s*[–-]\s*\1\.(\d+)\b/);
  if (range) {
    const [, prefix, a, b] = range;
    const ids = [];
    for (let n = Number(a); n <= Number(b); n++) ids.push(`${prefix}.${n}`);
    return ids;
  }
  const single = label.match(/^(\d+B?\.\d+)\b/);
  if (single) return [single[1]];
  const group = label.match(/^(\d+B?)\b/);
  if (group) return [`group:${group[1]}`];
  problems.push(`${ctx}: could not read related link "${label}"`);
  return [];
}

function parseRelated(cell, problems, ctx) {
  const labels = [...cell.matchAll(/\[([^\]]+)\]\([^)]*\)/g)].map((m) => m[1]);
  // Fallback for plain-text (unlinked) related cells
  const list = labels.length ? labels : cell.split(/,\s*(?=\d)/).filter(Boolean);
  return [...new Set(list.flatMap((l) => parseRelatedLabel(decode(l), problems, ctx)))];
}

function main() {
  const md = readFileSync(input, "utf8");
  const lines = logicalLines(md);
  const groups = [];
  const questions = [];
  const problems = [];
  let group = null;
  let header = null;

  for (const line of lines) {
    const heading = line.match(/^#{2,3}\s+(\d+B?)\.\s+(.+)$/);
    if (heading) {
      const [, id, title] = heading;
      const clean = decode(title).replace(/^"|"$/g, "");
      group = {
        id,
        slug: slugify(id === "3B" ? "missing books" : clean),
        title: clean,
        order: groups.length + 1,
      };
      groups.push(group);
      header = null;
      continue;
    }
    if (/^#{1,3}\s/.test(line)) {
      group = null;
      header = null;
      continue;
    } // non-numbered heading (e.g. "How to read this")
    if (!group) continue;

    if (line.startsWith("|")) {
      const cells = splitRow(line);
      if (cells.every((c) => /^:?-+:?$/.test(c))) continue; // separator row
      if (!header) {
        header = cells.map((c) => c.toLowerCase());
        continue;
      }

      const col = (name) => cells[header.findIndex((h) => h.startsWith(name))] ?? "";
      const id = decode(col("#"));
      const ctx = `Row ${id || "?"}`;
      const title = decode(col("question") || col("book"));
      const statusWord = decode(col("status")).toLowerCase();
      const answerCell = decode(col("basic answer") || col("what it is"));
      const [answer, ...noteParts] = answerCell.split(NOTE_SPLIT);
      const domainsCell = decode(col("domains"));

      if (!id) {
        problems.push(`${group.id}: row without an id`);
        continue;
      }
      if (!STATUS[statusWord]) problems.push(`${ctx}: unknown status "${statusWord}"`);
      if (!title) problems.push(`${ctx}: missing question title`);
      if (!answer.trim()) problems.push(`${ctx}: missing answer`);
      const domains = domainsCell ? domainsCell.split(/,\s*/).filter(Boolean) : [];
      for (const d of domains) if (!DOMAINS.has(d)) problems.push(`${ctx}: unknown domain "${d}"`);

      const q = {
        id,
        slug: slugify(`${id} ${title}`),
        groupId: group.id,
        title,
        status: STATUS[statusWord],
        domains,
        answer: answer.trim(),
        related: parseRelated(col("related"), problems, ctx),
        hasArticle: existsSync(articleFile(id)),
      };
      if (header.some((h) => h === "group")) q.bookGroup = decode(col("group"));
      const notes = noteParts.join(" ").trim();
      if (notes) q.authorNotes = notes;
      questions.push(q);
      continue;
    }

    // First plain paragraph under a group heading becomes its intro.
    const text = decode(line);
    if (text && !header && !group.intro) group.intro = text;
  }

  // Validation
  const ids = new Set(questions.map((q) => q.id));
  const groupIds = new Set(groups.map((g) => g.id));
  const seen = new Set();
  const seenSlugs = new Set();
  for (const q of questions) {
    if (seen.has(q.id)) problems.push(`Duplicate id ${q.id}`);
    seen.add(q.id);
    if (seenSlugs.has(q.slug)) problems.push(`Duplicate slug ${q.slug}`);
    seenSlugs.add(q.slug);
    for (const r of q.related) {
      const ok = r.startsWith("group:") ? groupIds.has(r.slice(6)) : ids.has(r);
      if (!ok) problems.push(`${q.id}: related "${r}" does not exist`);
      if (r === q.id) problems.push(`${q.id}: links to itself`);
    }
  }

  // Fail without writing, so a typo can never ship a guessed status or a broken link.
  if (problems.length) {
    console.error(`Import found ${problems.length} problem(s):\n- ` + problems.join("\n- "));
    console.error(`\n${output} was not updated.`);
    process.exit(1);
  }

  mkdirSync(dirname(output), { recursive: true });
  writeFileSync(
    output,
    JSON.stringify({ generatedFrom: input, groups, questions }, null, 2) + "\n",
  );
  const counts = questions.reduce((a, q) => ((a[q.status] = (a[q.status] ?? 0) + 1), a), {});
  const articles = questions.filter((q) => q.hasArticle).length;
  console.log(
    `Wrote ${groups.length} groups and ${questions.length} questions to ${output}`,
    counts,
    { articles },
  );
}

main();