tested/scripts/check-links.mjs

#!/usr/bin/env node
// Checks every external link in the articles (Learn more links and source URLs).
//
//   npm run check-links              check all articles
//   npm run check-links -- 2-1 7-9   check only these article files
//   npm run check-links -- --fresh   ignore the cache and re-check everything
//
// YouTube links are checked through YouTube's oEmbed endpoint, which confirms the video exists
// and returns its real title and channel, so you can see that each video is what it claims to be.
// Other links get a normal request. Results are cached in .cache/links.json for a week.
// Needs internet access, so it isn't part of `npm run build`.

import { readFileSync, readdirSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
import { parse } from "yaml";

const dir = "src/content/articles";
const cacheFile = ".cache/links.json";
const WEEK = 7 * 24 * 60 * 60 * 1000;
const args = process.argv.slice(2);
const fresh = args.includes("--fresh");
const only = args.filter((a) => !a.startsWith("--")).map((a) => a.replace(/\.mdx$/, ""));

const cache = !fresh && existsSync(cacheFile) ? JSON.parse(readFileSync(cacheFile, "utf8")) : {};

const frontmatter = (text) => parse(text.match(/^---\r?\n([\s\S]*?)\r?\n---/)?.[1] ?? "") ?? {};

// Collect links per article.
const jobs = [];
for (const file of readdirSync(dir).filter((f) => f.endsWith(".mdx"))) {
  const id = file.replace(/\.mdx$/, "");
  if (only.length && !only.includes(id)) continue;
  const data = frontmatter(readFileSync(`${dir}/${file}`, "utf8"));
  for (const l of data.learnMore ?? [])
    jobs.push({ file, url: l.url, title: l.title, type: l.type });
  for (const s of data.sources ?? [])
    if (s.url) jobs.push({ file, url: s.url, title: s.title, type: "source" });
}

const youtubeId = (url) =>
  url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([\w-]{11})/)?.[1];

const UA =
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130 Safari/537.36";

async function check(url) {
  const hit = cache[url];
  if (hit && Date.now() - hit.at < WEEK) return hit;
  let result;
  try {
    const id = youtubeId(url);
    if (id) {
      const res = await fetch(
        `https://www.youtube.com/oembed?url=${encodeURIComponent(`https://www.youtube.com/watch?v=${id}`)}&format=json`,
      );
      result = res.ok
        ? {
            ok: true,
            status: res.status,
            ...(({ title, author_name }) => ({ title, channel: author_name }))(await res.json()),
          }
        : { ok: false, status: res.status };
    } else {
      const res = await fetch(url, {
        redirect: "follow",
        headers: { "User-Agent": UA, Accept: "text/html,application/xhtml+xml,*/*" },
        signal: AbortSignal.timeout(20000),
      });
      // Some sites refuse automated requests (403/429) even though the page exists.
      result = {
        ok: res.ok,
        blocked: [401, 403, 429].includes(res.status),
        status: res.status,
        final: res.url,
      };
    }
  } catch (e) {
    result = { ok: false, status: 0, error: String(e.cause?.code ?? e.message) };
  }
  result.at = Date.now();
  cache[url] = result;
  return result;
}

// Check with limited concurrency.
const unique = [...new Set(jobs.map((j) => j.url))];
const queue = [...unique];
await Promise.all(
  Array.from({ length: 8 }, async () => {
    while (queue.length) await check(queue.shift());
  }),
);

mkdirSync(".cache", { recursive: true });
writeFileSync(cacheFile, JSON.stringify(cache, null, 2));

let broken = 0;
let blocked = 0;
const videos = [];
for (const j of jobs) {
  const r = cache[j.url];
  if (youtubeId(j.url) && r.ok)
    videos.push(`  ${j.file.padEnd(10)} “${r.title}” — ${r.channel}   (listed as: ${j.title})`);
  if (r.ok) continue;
  if (r.blocked) {
    blocked++;
    console.log(`?  ${j.file.padEnd(10)} ${r.status} (site refuses automated checks) ${j.url}`);
  } else {
    broken++;
    console.log(`✗  ${j.file.padEnd(10)} ${r.status || r.error} ${j.url}`);
  }
}
if (videos.length) console.log(`\nVideos (real titles from YouTube):\n${videos.join("\n")}`);
console.log(
  `\n${jobs.length} links in ${new Set(jobs.map((j) => j.file)).size} articles: ${broken} broken, ${blocked} could not be checked automatically.`,
);
process.exit(broken ? 1 : 0);