mytime/scripts/scrape-bridge-days.mjs

#!/usr/bin/env node
/**
 * Yearly bridge-days refresh from ferien-feiertage.de.
 *
 * Fetches `/bundesland/<slug>/<year>/brueckentage/` for every German state
 * and writes the result to `src/data/bridge-days.generated.ts`, which the
 * Brückentage tab reads (offline-friendly, bundled at build time).
 *
 * Run once a year, e.g.:
 *   npm run scrape:bridge
 *   npm run scrape:bridge -- --years=2027 --states=HH,BY
 *
 * Please be polite: the script pauses between requests and sends an
 * identifying User-Agent. robots.txt currently allows crawling (/).
 */
import { readFileSync, writeFileSync } from "node:fs";
import { parseBridgeDaysTable } from "./parse-bridge-days.mjs";

const BASE = "https://ferien-feiertage.de";
const USER_AGENT =
  "MyUrlaubZeit/1.0 (personal holiday planner; yearly refresh; contact via repo owner)";
const OUT = new URL("../src/data/bridge-days.generated.ts", import.meta.url);
const STORE = new URL("../src/data/bridge-days.json", import.meta.url);

const STATES = [
  ["BW", "Baden-Württemberg"],
  ["BY", "Bayern"],
  ["BE", "Berlin"],
  ["BB", "Brandenburg"],
  ["HB", "Bremen"],
  ["HH", "Hamburg"],
  ["HE", "Hessen"],
  ["MV", "Mecklenburg-Vorpommern"],
  ["NI", "Niedersachsen"],
  ["NW", "Nordrhein-Westfalen"],
  ["RP", "Rheinland-Pfalz"],
  ["SL", "Saarland"],
  ["SN", "Sachsen"],
  ["ST", "Sachsen-Anhalt"],
  ["SH", "Schleswig-Holstein"],
  ["TH", "Thüringen"],
];

export function slugOf(name) {
  return name
    .toLowerCase()
    .replace(/ä/g, "ae")
    .replace(/ö/g, "oe")
    .replace(/ü/g, "ue")
    .replace(/ß/g, "ss")
    .replace(/\s+/g, "-");
}

function parseArgs(argv) {
  const args = { years: null, states: null, delay: 600 };
  for (const arg of argv) {
    if (arg.startsWith("--years=")) {
      args.years = arg.slice("--years=".length).split(",").map(Number);
    } else if (arg.startsWith("--states=")) {
      args.states = arg.slice("--states=".length).split(",").map((s) => s.toUpperCase());
    } else if (arg.startsWith("--delay=")) {
      args.delay = Number(arg.slice("--delay=".length));
    }
  }
  return args;
}

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

async function fetchTable(slug, year) {
  const url = `${BASE}/bundesland/${slug}/${year}/brueckentage/`;
  const response = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
  if (!response.ok) {
    throw new Error(`HTTP ${response.status} for ${url}`);
  }
  return parseBridgeDaysTable(await response.text(), year);
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const now = new Date().getFullYear();
  const years = args.years ?? [now, now + 1];
  const states = args.states
    ? STATES.filter(([code]) => args.states.includes(code))
    : STATES;
  if (states.length === 0) {
    throw new Error("No matching states");
  }
  const data = {};
  try {
    Object.assign(data, JSON.parse(readFileSync(STORE, "utf-8")));
    console.log(`merged ${Object.keys(data).length} existing keys`);
  } catch {
    console.log("no existing store, starting fresh");
  }
  const missing = [];
  for (const [code, name] of states) {
    const slug = slugOf(name);
    for (const year of years) {
      const key = `${code}-${year}`;
      try {
        data[key] = await fetchTable(slug, year);
        console.log(`ok   ${key}: ${data[key].length} entries`);
      } catch (error) {
        missing.push(`${key} (${error.message})`);
        console.error(`fail ${key}: ${error.message}`);
      }
      await sleep(args.delay);
    }
  }
  const fetchedAt = new Date().toISOString();
  writeFileSync(STORE, JSON.stringify(data, null, 2) + "\n");
  const storedYears = [...new Set(Object.keys(data).map((key) => Number(key.slice(-4))))].sort();
  const storedStates = [...new Set(Object.keys(data).map((key) => key.slice(0, -5)))].sort();
  const payload =
    `// AUTO-GENERATED by scripts/scrape-bridge-days.mjs — do not edit by hand.\n` +
    `// Source: ${BASE} (robots.txt allows crawling). Refresh yearly via \`npm run scrape:bridge\`.\n` +
    `export interface ScrapedBridgeDay {\n` +
    `  start: string;\n` +
    `  end: string;\n` +
    `  occasion: string;\n` +
    `  vacationDays: number;\n` +
    `  freeDays: number;\n` +
    `  freeStart: string;\n` +
    `  freeEnd: string;\n` +
    `}\n\n` +
    `export const SCRAPED_BRIDGE_META = ${JSON.stringify({ source: BASE, fetchedAt, years: storedYears, states: storedStates }, null, 2)} as const;\n\n` +
    `export const SCRAPED_BRIDGE_DAYS: Record<string, ScrapedBridgeDay[]> = ${JSON.stringify(data, null, 2)};\n`;
  writeFileSync(OUT, payload);
  console.log(`wrote ${OUT.pathname} (${Object.keys(data).length} keys)`);
  if (missing.length > 0) {
    console.error(`missing: ${missing.join("; ")}`);
    process.exitCode = 1;
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  await main();
}