test(wasm): gate and report the WASM ABI apart from Rust coverage

.github/workflows/publish.yml

@@ -814,6 +814,12 @@ jobs:
           RUSTUP_TOOLCHAIN: 1.99.1
       - name: Regression test (bun-plugin preload race)
         run: bun run --filter @devup-ui/bun-plugin test:regression
+      # Rust coverage excludes every wasm-bindgen wrapper, so the WASM ABI is
+      # checked here instead: the public corpus must match the native addon
+      # byte for byte, every invalid argument shape must be rejected, and the
+      # step summary reports which exports and positions were reached.
+      - name: WASM ABI coverage
+        run: bun run test:wasm-abi
       - name: Format Rollback
         run: |
           rm -rf .rustfmt.toml

e2e/binding/wasm-abi/coverage.mjs

@@ -0,0 +1,157 @@
+/**
+ * ABI coverage of the generated WASM package, reported apart from the Rust
+ * line coverage: every wrapper tarpaulin skips (`cfg(not(tarpaulin_include))`)
+ * is only reached through these JS calls.
+ */
+
+/** Parameters the boundary deliberately leaves unvalidated, with the reason. */
+export const UNVALIDATED = {
+  'hasDevupUI.import_aliases':
+    'only read to find aliased imports; bindings/argument-validation.cjs never declared it',
+}
+
+/** Parse `export function name(a: T, b?: U): R;` lines of a wasm-bindgen `.d.ts`. */
+export function parseDeclarations(dts) {
+  const functions = {}
+  for (const [, name, list] of dts.matchAll(
+    /^export function (\w+)\(([^)]*)\)/gm,
+  )) {
+    functions[name] = list
+      .split(',')
+      .map((parameter) => parameter.trim())
+      .filter(Boolean)
+      .map((parameter) => parameter.replace(/\??:.*$/s, ''))
+  }
+  return functions
+}
+
+/**
+ * Members of the generated `Output` class: its getters, `free` and
+ * `Symbol.dispose`, named as the public scenarios label their steps.
+ */
+export function parseOutputMembers(dts) {
+  const body = /^export class Output \{([\s\S]*?)^\}/m.exec(dts)?.[1] ?? ''
+  return [
+    ...body.matchAll(
+      /^\s*(?:readonly (\w+):|(\w+)\(\)|\[(Symbol\.\w+)\]\(\))/gm,
+    ),
+  ].map(([, getter, method, symbol]) =>
+    symbol ? `Symbol(${symbol})` : (getter ?? method),
+  )
+}
+
+/**
+ * Count how the public scenarios reached each export: as a recorded step, or
+ * as one of the observers every step snapshots before and after it runs.
+ */
+export function countScenarioCalls(records) {
+  const calls = {}
+  const add = (name, count = 1) => (calls[name] = (calls[name] ?? 0) + count)
+  for (const record of records) {
+    if (record.kind !== 'step') continue
+    add(record.label)
+    for (const state of [record.before, record.after]) {
+      for (const name of Object.keys(state.exports)) add(name)
+      for (const name of Object.keys(state.config)) add(name)
+      add('getThemeInterface')
+      add('getCss', state.css.length)
+    }
+  }
+  return calls
+}
+
+/**
+ * Join the declarations, the scenario calls and the rejection cases into one
+ * report. `ok` is false when an export is never called, or when an argument
+ * position has no rejection case and is not listed in UNVALIDATED.
+ */
+export function evaluate({ declarations, members, calls, cases }) {
+  const rejected = {}
+  for (const test of cases) {
+    const key = `${test.fn}.${test.parameter}`
+    rejected[key] = (rejected[key] ?? 0) + 1
+  }
+  const problems = []
+  const rows = Object.entries(declarations)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([name, parameters]) => {
+      const called = calls[name] ?? 0
+      if (called === 0) problems.push(`${name}: never called`)
+      const positions = parameters.map((parameter, index) => {
+        const key = `${name}.${parameter}`
+        const count = rejected[key] ?? 0
+        const position = cases.find(
+          (test) => test.fn === name && test.parameter === parameter,
+        )
+        if (position && position.index !== index)
+          problems.push(
+            `${key}: declared at ${index}, validated at ${position.index}`,
+          )
+        if (count === 0 && !Object.hasOwn(UNVALIDATED, key))
+          problems.push(`${key}: no rejection case`)
+        return { parameter, rejected: count, exempt: count === 0 }
+      })
+      return { name, called, positions }
+    })
+  // The scenarios read every member on a live output and again after release.
+  for (const member of members) {
+    const called = ['live', 'release', 'released']
+      .map((stage) => calls[`${stage}-${member}`] ?? 0)
+      .reduce((sum, count) => sum + count, 0)
+    if (called === 0) problems.push(`Output.${member}: never called`)
+    rows.push({ name: `Output.${member}`, called, positions: [] })
+  }
+  for (const key of Object.keys(rejected)) {
+    const [name, parameter] = key.split('.')
+    if (!declarations[name]?.includes(parameter))
+      problems.push(`${key}: rejection case for an undeclared parameter`)
+  }
+  for (const key of Object.keys(UNVALIDATED)) {
+    const [name, parameter] = key.split('.')
+    if (!declarations[name]?.includes(parameter) || rejected[key])
+      problems.push(`${key}: stale UNVALIDATED entry`)
+  }
+  const positions = rows.flatMap((row) => row.positions)
+  return {
+    exports: rows.length,
+    called: rows.filter((row) => row.called > 0).length,
+    positions: positions.length,
+    validated: positions.filter((position) => !position.exempt).length,
+    cases: cases.length,
+    rows,
+    problems,
+    ok: problems.length === 0,
+  }
+}
+
+/** Render the report as Markdown for a CI step summary. */
+export function report(result) {
+  const lines = [
+    '### WASM ABI coverage',
+    '',
+    `${result.called}/${result.exports} exports and \`Output\` members called, ${result.validated}/${result.positions} argument positions with rejection cases (${result.cases} cases).`,
+    '',
+    '| Export | Calls | Rejection cases per argument |',
+    '| --- | ---: | --- |',
+    ...result.rows.map(
+      (row) =>
+        `| \`${row.name}\` | ${row.called} | ${
+          row.positions
+            .map(
+              (position) =>
+                `${position.parameter}: ${position.exempt ? 'unvalidated' : position.rejected}`,
+            )
+            .join(', ') || '-'
+        } |`,
+    ),
+    '',
+  ]
+  if (result.problems.length)
+    lines.push(
+      '**Problems**',
+      '',
+      ...result.problems.map((problem) => `- ${problem}`),
+      '',
+    )
+  return lines.join('\n')
+}

e2e/binding/wasm-abi/coverage.test.mjs

@@ -0,0 +1,162 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { cases } from '../validation-cases.mjs'
+import {
+  countScenarioCalls,
+  evaluate,
+  parseDeclarations,
+  parseOutputMembers,
+  report,
+  UNVALIDATED,
+} from './coverage.mjs'
+
+const dts = `/* tslint:disable */
+export class Output {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * Get the code
+     */
+    readonly code: string;
+    readonly css: string | undefined;
+}
+
+export function getCss(file_num: number | null | undefined, import_main_css: boolean): string;
+
+/**
+ * Docs mentioning export function fake(a: string) stay out.
+ */
+export function setPrefix(prefix?: string | null): void;
+
+export function isDebug(): boolean;
+`
+
+const state = (exports, config, css) => ({
+  exports: Object.fromEntries(exports.map((name) => [name, {}])),
+  config: Object.fromEntries(config.map((name) => [name, {}])),
+  interface: {},
+  css: Array.from({ length: css }, () => ({})),
+})
+
+test('parses exported functions with their parameter names', () => {
+  assert.deepEqual(parseDeclarations(dts), {
+    getCss: ['file_num', 'import_main_css'],
+    setPrefix: ['prefix'],
+    isDebug: [],
+  })
+})
+
+test('parses the Output members as the scenarios label them', () => {
+  assert.deepEqual(parseOutputMembers(dts), [
+    'free',
+    'Symbol(Symbol.dispose)',
+    'code',
+    'css',
+  ])
+  assert.deepEqual(parseOutputMembers('export function a(): void;'), [])
+})
+
+test('counts steps and the observers each step snapshots', () => {
+  const snapshot = state(['exportSheet'], ['isDebug'], 2)
+  const calls = countScenarioCalls([
+    { kind: 'note', label: 'ignored' },
+    { kind: 'start', state: snapshot },
+    { kind: 'step', label: 'setPrefix', before: snapshot, after: snapshot },
+    { kind: 'step', label: 'setPrefix', before: snapshot, after: snapshot },
+  ])
+  assert.deepEqual(calls, {
+    setPrefix: 2,
+    exportSheet: 4,
+    isDebug: 4,
+    getThemeInterface: 4,
+    getCss: 8,
+  })
+})
+
+const fixture = (overrides = {}) => ({
+  declarations: {
+    getCss: ['file_num'],
+    hasDevupUI: ['import_aliases'],
+    isDebug: [],
+  },
+  members: ['code'],
+  calls: { getCss: 1, hasDevupUI: 1, isDebug: 1, 'live-code': 1 },
+  cases: [{ fn: 'getCss', parameter: 'file_num', index: 0 }],
+  ...overrides,
+})
+
+test('passes when every export and member is called and every argument rejected', () => {
+  const result = evaluate(fixture())
+  assert.equal(result.ok, true)
+  assert.deepEqual(result.problems, [])
+  assert.equal(result.exports, 4)
+  assert.equal(result.called, 4)
+  assert.equal(result.positions, 2)
+  assert.equal(result.validated, 1)
+  assert.deepEqual(
+    result.rows.map((row) => [row.name, row.called]),
+    [
+      ['getCss', 1],
+      ['hasDevupUI', 1],
+      ['isDebug', 1],
+      ['Output.code', 1],
+    ],
+  )
+  const markdown = report(result)
+  assert.match(markdown, /4\/4 exports and `Output` members called/)
+  assert.match(markdown, /1\/2 argument positions with rejection cases/)
+  assert.match(markdown, /\| `getCss` \| 1 \| file_num: 1 \|/)
+  assert.match(
+    markdown,
+    /\| `hasDevupUI` \| 1 \| import_aliases: unvalidated \|/,
+  )
+  assert.match(markdown, /\| `isDebug` \| 1 \| - \|/)
+  assert.doesNotMatch(markdown, /Problems/)
+})
+
+test('fails on uncalled exports, unvalidated or moved arguments and stale cases', () => {
+  const result = evaluate(
+    fixture({
+      declarations: { getCss: ['index', 'file_num'], isDebug: [] },
+      calls: { getCss: 1 },
+      cases: [
+        { fn: 'getCss', parameter: 'file_num', index: 0 },
+        { fn: 'removed', parameter: 'value', index: 0 },
+      ],
+    }),
+  )
+  assert.equal(result.ok, false)
+  assert.deepEqual(result.problems, [
+    'getCss.index: no rejection case',
+    'getCss.file_num: declared at 1, validated at 0',
+    'isDebug: never called',
+    'Output.code: never called',
+    'removed.value: rejection case for an undeclared parameter',
+    'hasDevupUI.import_aliases: stale UNVALIDATED entry',
+  ])
+  const markdown = report(result)
+  assert.match(markdown, /index: unvalidated, file_num: 1/)
+  assert.match(markdown, /- isDebug: never called/)
+})
+
+test('exempts only the documented unvalidated parameters', () => {
+  const result = evaluate(
+    fixture({
+      declarations: { hasDevupUI: ['filename', 'import_aliases'] },
+      members: [],
+      calls: { hasDevupUI: 1 },
+      cases: [{ fn: 'hasDevupUI', parameter: 'filename', index: 0 }],
+    }),
+  )
+  assert.deepEqual(result.problems, [])
+  assert.ok(UNVALIDATED['hasDevupUI.import_aliases'])
+  assert.match(report(result), /import_aliases: unvalidated/)
+})
+
+test('the validation oracle covers every documented argument position', () => {
+  const positions = new Set(cases.map((item) => `${item.fn}.${item.parameter}`))
+  assert.equal(positions.size, 45)
+  for (const key of Object.keys(UNVALIDATED)) assert.ok(!positions.has(key))
+})