diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9393231bb..a11fc0dc4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -814,6 +814,12 @@ jobs: RUSTUP_TOOLCHAIN: 1.99.1 - name: Regression test (bun-plugin preload race) run: bun run --filter @devup-ui/bun-plugin test:regression + # Rust coverage excludes every wasm-bindgen wrapper, so the WASM ABI is + # checked here instead: the public corpus must match the native addon + # byte for byte, every invalid argument shape must be rejected, and the + # step summary reports which exports and positions were reached. + - name: WASM ABI coverage + run: bun run test:wasm-abi - name: Format Rollback run: | rm -rf .rustfmt.toml diff --git a/e2e/binding/wasm-abi/coverage.mjs b/e2e/binding/wasm-abi/coverage.mjs new file mode 100644 index 000000000..151bbf651 --- /dev/null +++ b/e2e/binding/wasm-abi/coverage.mjs @@ -0,0 +1,157 @@ +/** + * ABI coverage of the generated WASM package, reported apart from the Rust + * line coverage: every wrapper tarpaulin skips (`cfg(not(tarpaulin_include))`) + * is only reached through these JS calls. + */ + +/** Parameters the boundary deliberately leaves unvalidated, with the reason. */ +export const UNVALIDATED = { + 'hasDevupUI.import_aliases': + 'only read to find aliased imports; bindings/argument-validation.cjs never declared it', +} + +/** Parse `export function name(a: T, b?: U): R;` lines of a wasm-bindgen `.d.ts`. */ +export function parseDeclarations(dts) { + const functions = {} + for (const [, name, list] of dts.matchAll( + /^export function (\w+)\(([^)]*)\)/gm, + )) { + functions[name] = list + .split(',') + .map((parameter) => parameter.trim()) + .filter(Boolean) + .map((parameter) => parameter.replace(/\??:.*$/s, '')) + } + return functions +} + +/** + * Members of the generated `Output` class: its getters, `free` and + * `Symbol.dispose`, named as the public scenarios label their steps. + */ +export function parseOutputMembers(dts) { + const body = /^export class Output \{([\s\S]*?)^\}/m.exec(dts)?.[1] ?? '' + return [ + ...body.matchAll( + /^\s*(?:readonly (\w+):|(\w+)\(\)|\[(Symbol\.\w+)\]\(\))/gm, + ), + ].map(([, getter, method, symbol]) => + symbol ? `Symbol(${symbol})` : (getter ?? method), + ) +} + +/** + * Count how the public scenarios reached each export: as a recorded step, or + * as one of the observers every step snapshots before and after it runs. + */ +export function countScenarioCalls(records) { + const calls = {} + const add = (name, count = 1) => (calls[name] = (calls[name] ?? 0) + count) + for (const record of records) { + if (record.kind !== 'step') continue + add(record.label) + for (const state of [record.before, record.after]) { + for (const name of Object.keys(state.exports)) add(name) + for (const name of Object.keys(state.config)) add(name) + add('getThemeInterface') + add('getCss', state.css.length) + } + } + return calls +} + +/** + * Join the declarations, the scenario calls and the rejection cases into one + * report. `ok` is false when an export is never called, or when an argument + * position has no rejection case and is not listed in UNVALIDATED. + */ +export function evaluate({ declarations, members, calls, cases }) { + const rejected = {} + for (const test of cases) { + const key = `${test.fn}.${test.parameter}` + rejected[key] = (rejected[key] ?? 0) + 1 + } + const problems = [] + const rows = Object.entries(declarations) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, parameters]) => { + const called = calls[name] ?? 0 + if (called === 0) problems.push(`${name}: never called`) + const positions = parameters.map((parameter, index) => { + const key = `${name}.${parameter}` + const count = rejected[key] ?? 0 + const position = cases.find( + (test) => test.fn === name && test.parameter === parameter, + ) + if (position && position.index !== index) + problems.push( + `${key}: declared at ${index}, validated at ${position.index}`, + ) + if (count === 0 && !Object.hasOwn(UNVALIDATED, key)) + problems.push(`${key}: no rejection case`) + return { parameter, rejected: count, exempt: count === 0 } + }) + return { name, called, positions } + }) + // The scenarios read every member on a live output and again after release. + for (const member of members) { + const called = ['live', 'release', 'released'] + .map((stage) => calls[`${stage}-${member}`] ?? 0) + .reduce((sum, count) => sum + count, 0) + if (called === 0) problems.push(`Output.${member}: never called`) + rows.push({ name: `Output.${member}`, called, positions: [] }) + } + for (const key of Object.keys(rejected)) { + const [name, parameter] = key.split('.') + if (!declarations[name]?.includes(parameter)) + problems.push(`${key}: rejection case for an undeclared parameter`) + } + for (const key of Object.keys(UNVALIDATED)) { + const [name, parameter] = key.split('.') + if (!declarations[name]?.includes(parameter) || rejected[key]) + problems.push(`${key}: stale UNVALIDATED entry`) + } + const positions = rows.flatMap((row) => row.positions) + return { + exports: rows.length, + called: rows.filter((row) => row.called > 0).length, + positions: positions.length, + validated: positions.filter((position) => !position.exempt).length, + cases: cases.length, + rows, + problems, + ok: problems.length === 0, + } +} + +/** Render the report as Markdown for a CI step summary. */ +export function report(result) { + const lines = [ + '### WASM ABI coverage', + '', + `${result.called}/${result.exports} exports and \`Output\` members called, ${result.validated}/${result.positions} argument positions with rejection cases (${result.cases} cases).`, + '', + '| Export | Calls | Rejection cases per argument |', + '| --- | ---: | --- |', + ...result.rows.map( + (row) => + `| \`${row.name}\` | ${row.called} | ${ + row.positions + .map( + (position) => + `${position.parameter}: ${position.exempt ? 'unvalidated' : position.rejected}`, + ) + .join(', ') || '-' + } |`, + ), + '', + ] + if (result.problems.length) + lines.push( + '**Problems**', + '', + ...result.problems.map((problem) => `- ${problem}`), + '', + ) + return lines.join('\n') +} diff --git a/e2e/binding/wasm-abi/coverage.test.mjs b/e2e/binding/wasm-abi/coverage.test.mjs new file mode 100644 index 000000000..54d0a206c --- /dev/null +++ b/e2e/binding/wasm-abi/coverage.test.mjs @@ -0,0 +1,162 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { cases } from '../validation-cases.mjs' +import { + countScenarioCalls, + evaluate, + parseDeclarations, + parseOutputMembers, + report, + UNVALIDATED, +} from './coverage.mjs' + +const dts = `/* tslint:disable */ +export class Output { + private constructor(); + free(): void; + [Symbol.dispose](): void; + /** + * Get the code + */ + readonly code: string; + readonly css: string | undefined; +} + +export function getCss(file_num: number | null | undefined, import_main_css: boolean): string; + +/** + * Docs mentioning export function fake(a: string) stay out. + */ +export function setPrefix(prefix?: string | null): void; + +export function isDebug(): boolean; +` + +const state = (exports, config, css) => ({ + exports: Object.fromEntries(exports.map((name) => [name, {}])), + config: Object.fromEntries(config.map((name) => [name, {}])), + interface: {}, + css: Array.from({ length: css }, () => ({})), +}) + +test('parses exported functions with their parameter names', () => { + assert.deepEqual(parseDeclarations(dts), { + getCss: ['file_num', 'import_main_css'], + setPrefix: ['prefix'], + isDebug: [], + }) +}) + +test('parses the Output members as the scenarios label them', () => { + assert.deepEqual(parseOutputMembers(dts), [ + 'free', + 'Symbol(Symbol.dispose)', + 'code', + 'css', + ]) + assert.deepEqual(parseOutputMembers('export function a(): void;'), []) +}) + +test('counts steps and the observers each step snapshots', () => { + const snapshot = state(['exportSheet'], ['isDebug'], 2) + const calls = countScenarioCalls([ + { kind: 'note', label: 'ignored' }, + { kind: 'start', state: snapshot }, + { kind: 'step', label: 'setPrefix', before: snapshot, after: snapshot }, + { kind: 'step', label: 'setPrefix', before: snapshot, after: snapshot }, + ]) + assert.deepEqual(calls, { + setPrefix: 2, + exportSheet: 4, + isDebug: 4, + getThemeInterface: 4, + getCss: 8, + }) +}) + +const fixture = (overrides = {}) => ({ + declarations: { + getCss: ['file_num'], + hasDevupUI: ['import_aliases'], + isDebug: [], + }, + members: ['code'], + calls: { getCss: 1, hasDevupUI: 1, isDebug: 1, 'live-code': 1 }, + cases: [{ fn: 'getCss', parameter: 'file_num', index: 0 }], + ...overrides, +}) + +test('passes when every export and member is called and every argument rejected', () => { + const result = evaluate(fixture()) + assert.equal(result.ok, true) + assert.deepEqual(result.problems, []) + assert.equal(result.exports, 4) + assert.equal(result.called, 4) + assert.equal(result.positions, 2) + assert.equal(result.validated, 1) + assert.deepEqual( + result.rows.map((row) => [row.name, row.called]), + [ + ['getCss', 1], + ['hasDevupUI', 1], + ['isDebug', 1], + ['Output.code', 1], + ], + ) + const markdown = report(result) + assert.match(markdown, /4\/4 exports and `Output` members called/) + assert.match(markdown, /1\/2 argument positions with rejection cases/) + assert.match(markdown, /\| `getCss` \| 1 \| file_num: 1 \|/) + assert.match( + markdown, + /\| `hasDevupUI` \| 1 \| import_aliases: unvalidated \|/, + ) + assert.match(markdown, /\| `isDebug` \| 1 \| - \|/) + assert.doesNotMatch(markdown, /Problems/) +}) + +test('fails on uncalled exports, unvalidated or moved arguments and stale cases', () => { + const result = evaluate( + fixture({ + declarations: { getCss: ['index', 'file_num'], isDebug: [] }, + calls: { getCss: 1 }, + cases: [ + { fn: 'getCss', parameter: 'file_num', index: 0 }, + { fn: 'removed', parameter: 'value', index: 0 }, + ], + }), + ) + assert.equal(result.ok, false) + assert.deepEqual(result.problems, [ + 'getCss.index: no rejection case', + 'getCss.file_num: declared at 1, validated at 0', + 'isDebug: never called', + 'Output.code: never called', + 'removed.value: rejection case for an undeclared parameter', + 'hasDevupUI.import_aliases: stale UNVALIDATED entry', + ]) + const markdown = report(result) + assert.match(markdown, /index: unvalidated, file_num: 1/) + assert.match(markdown, /- isDebug: never called/) +}) + +test('exempts only the documented unvalidated parameters', () => { + const result = evaluate( + fixture({ + declarations: { hasDevupUI: ['filename', 'import_aliases'] }, + members: [], + calls: { hasDevupUI: 1 }, + cases: [{ fn: 'hasDevupUI', parameter: 'filename', index: 0 }], + }), + ) + assert.deepEqual(result.problems, []) + assert.ok(UNVALIDATED['hasDevupUI.import_aliases']) + assert.match(report(result), /import_aliases: unvalidated/) +}) + +test('the validation oracle covers every documented argument position', () => { + const positions = new Set(cases.map((item) => `${item.fn}.${item.parameter}`)) + assert.equal(positions.size, 45) + for (const key of Object.keys(UNVALIDATED)) assert.ok(!positions.has(key)) +}) diff --git a/e2e/binding/wasm-abi/run.mjs b/e2e/binding/wasm-abi/run.mjs new file mode 100644 index 000000000..4d7e382c1 --- /dev/null +++ b/e2e/binding/wasm-abi/run.mjs @@ -0,0 +1,82 @@ +import { spawn } from 'node:child_process' +import { + appendFileSync, + mkdtempSync, + readFileSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { cases } from '../validation-cases.mjs' +import { + countScenarioCalls, + evaluate, + parseDeclarations, + parseOutputMembers, + report, +} from './coverage.mjs' + +// Runs the existing public-scenario and argument-validation oracles against +// the built WASM package, next to the native addon they must match, then +// reports which exports and argument positions they reached. +const here = fileURLToPath(new URL('.', import.meta.url)) +const root = resolve(here, '../../..') +const [ + wasm = join(root, 'bindings/devup-ui-wasm/pkg/index.js'), + native = join(root, 'bindings/devup-ui-binding/pkg/index.cjs'), +] = process.argv.slice(2) +const evidence = mkdtempSync(join(tmpdir(), 'devup-wasm-abi-')) + +function run(script, directory) { + const args = [ + join(here, '..', script), + wasm, + native, + join(evidence, directory), + ] + return new Promise((done) => { + const child = spawn(process.execPath, args, { + env: { ...process.env, NODE_OPTIONS: '', BUN_OPTIONS: '' }, + stdio: ['ignore', 'inherit', 'inherit'], + }) + child.on('error', (error) => done({ script, error: error.message })) + child.on('exit', (status, signal) => done({ script, status, signal })) + }) +} + +// identity.mjs: the public corpus on both engines, three times each, compared +// byte for byte. validation.mjs: every invalid argument shape on both engines +// rejected with the same TypeError and no state change. +const runs = await Promise.all([ + run('identity.mjs', 'identity'), + run('validation.mjs', 'validation'), +]) +const failed = runs.filter((result) => result.status !== 0) + +const records = readFileSync( + join(evidence, 'identity/wasm-1/records.jsonl'), + 'utf8', +) + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)) +const dts = readFileSync(join(dirname(wasm), 'index.d.ts'), 'utf8') +const result = evaluate({ + declarations: parseDeclarations(dts), + members: parseOutputMembers(dts), + calls: countScenarioCalls(records), + cases, +}) +const markdown = report(result) +console.info(markdown) +writeFileSync( + join(evidence, 'abi-coverage.json'), + `${JSON.stringify({ runs, ...result }, null, 2)}\n`, +) +if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown) +console.info(`Evidence: ${evidence}`) +for (const { script } of failed) console.error(`${script} failed`) +if (failed.length || !result.ok) process.exitCode = 1 diff --git a/package.json b/package.json index 410defde0..3e0b3d958 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "pack:native": "node e2e/binding/package/collect.mjs", "test:native:package": "node --test e2e/binding/package/package.test.mjs e2e/binding/package/publish.test.mjs e2e/binding/package/elf.test.mjs e2e/binding/package/build-host.test.mjs", "test:native:consumer": "node e2e/binding/package/consumer.mjs", + "test:wasm-abi": "node --test e2e/binding/wasm-abi/coverage.test.mjs && node e2e/binding/wasm-abi/run.mjs", "test:e2e:ui": "bunx playwright test --ui", "test:e2e:update": "bunx playwright test --update-snapshots", "build": "node e2e/binding/package/build-host.mjs && bun run --filter @devup-ui/wasm --filter @devup-ui/plugin-utils build && bun run --filter @devup-ui/react --filter @devup-ui/webpack-plugin build && bun run --filter @devup-ui/eslint-plugin --filter @devup-ui/vite-plugin --filter @devup-ui/next-plugin --filter @devup-ui/rsbuild-plugin --filter @devup-ui/bun-plugin --filter @devup-ui/components --filter @devup-ui/reset-css build",