From edef1cecbf1e0e7daa20ebd6aa83b180438bf0f4 Mon Sep 17 00:00:00 2001 From: znetsixe Date: Tue, 19 May 2026 10:13:49 +0200 Subject: [PATCH] tools: add output-manifest-verify; extend flow-lint with fan-out checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tools/output-manifest-verify/ — enforces .claude/rules/output-coverage.md §3: every node ships test/_output-manifest.md and every declared key is referenced by at least one test file. First run shows only machineGroupControl has the manifest (16 keys covered); all other nodes warn. --strict escalates "missing manifest" to an error for CI gating. - flow-lint gains two rules from the same output-coverage rule: * FN_OUTPUT_WIRES_MISMATCH — function declares outputs=N but wires has M arrays (causes silent dropped or duplicate emissions). * FN_PAYLOAD_NULL_LITERAL — function source contains `payload: null` literal (the η-null ui-chart crash pattern from 2026-05-14). First run found 1 instance in mgc/02-Dashboard.json. Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/flow-lint/bin/flow-lint.js | 23 +++ tools/output-manifest-verify/README.md | 47 ++++++ .../bin/output-manifest-verify.js | 148 ++++++++++++++++++ tools/output-manifest-verify/package.json | 13 ++ 4 files changed, 231 insertions(+) create mode 100644 tools/output-manifest-verify/README.md create mode 100644 tools/output-manifest-verify/bin/output-manifest-verify.js create mode 100644 tools/output-manifest-verify/package.json diff --git a/tools/flow-lint/bin/flow-lint.js b/tools/flow-lint/bin/flow-lint.js index b0bcdd3..dee2c85 100644 --- a/tools/flow-lint/bin/flow-lint.js +++ b/tools/flow-lint/bin/flow-lint.js @@ -35,6 +35,7 @@ function lintFlow(flowPath) { checkLinkPair(n, byId, findings); checkDebugLog(n, findings); checkBackwardWires(n, findings); + checkFunctionFanOut(n, findings); } checkGroupWidths(nodes, findings); return findings; @@ -172,6 +173,28 @@ function checkBackwardWires(n, findings) { } } +function checkFunctionFanOut(n, findings) { + if (n.type !== 'function') return; + const outputs = Number(n.outputs); + if (!Number.isFinite(outputs) || outputs <= 1) return; + if (!Array.isArray(n.wires) || n.wires.length !== outputs) { + findings.push({ + rule: 'FN_OUTPUT_WIRES_MISMATCH', + severity: 'error', + node: n.id, + msg: `function "${n.name || n.id}" declares outputs=${outputs} but wires has ${Array.isArray(n.wires) ? n.wires.length : 'no'} arrays.`, + }); + } + if (typeof n.func === 'string' && /payload\s*:\s*null\b/.test(n.func)) { + findings.push({ + rule: 'FN_PAYLOAD_NULL_LITERAL', + severity: 'error', + node: n.id, + msg: `function "${n.name || n.id}" emits a literal { payload: null } — crashes ui-chart on first frame; return the whole msg as null instead so the port skips.`, + }); + } +} + function checkGroupWidths(nodes, findings) { const pages = new Map(); for (const n of nodes) { diff --git a/tools/output-manifest-verify/README.md b/tools/output-manifest-verify/README.md new file mode 100644 index 0000000..94ce5e4 --- /dev/null +++ b/tools/output-manifest-verify/README.md @@ -0,0 +1,47 @@ +# @evolv/output-manifest-verify + +Enforce `.claude/rules/output-coverage.md` §3: + +1. Every node ships `test/_output-manifest.md`. +2. Every key declared in the manifest is referenced by at least one test + file under `test/**/*.test.js`. + +Designed to replace the manual verification checklist in the rule. The +rule itself can become a thin pointer to this tool once every node is in +compliance. + +## Usage + +```bash +# scan every node with a CONTRACT.md +node tools/output-manifest-verify/bin/output-manifest-verify.js + +# one node +node tools/output-manifest-verify/bin/output-manifest-verify.js nodes/machineGroupControl + +# fail (exit 1) when a manifest is missing, not just warn +node tools/output-manifest-verify/bin/output-manifest-verify.js --strict + +# CI / JSON output (one line per node) +node tools/output-manifest-verify/bin/output-manifest-verify.js --json +``` + +## Severity + +| Condition | Severity | +|---|---| +| `test/_output-manifest.md` missing | `warn` (escalates to `error` under `--strict`) | +| Manifest has no `##` sections / no tables | `error` | +| Declared key not referenced by any test file | `warn` | +| Manifest declares keys but `test/**/*.test.js` directory empty | `error` | + +## What it does NOT check (yet) + +- Whether each key has both a **populated** and **degraded** test (the + rule's §3 promise). Reaching that level of confidence requires + instrumenting the actual node runtime; tracked as a follow-up. +- The Port 1 (InfluxDB) cardinality discipline — see + `.claude/rules/telemetry.md`. + +Run after touching `_output-manifest.md`, `test/**/output-*.test.js`, +or any Port 0/1/2 emission key. diff --git a/tools/output-manifest-verify/bin/output-manifest-verify.js b/tools/output-manifest-verify/bin/output-manifest-verify.js new file mode 100644 index 0000000..6b2ffb3 --- /dev/null +++ b/tools/output-manifest-verify/bin/output-manifest-verify.js @@ -0,0 +1,148 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +function parseManifest(manifestPath) { + if (!fs.existsSync(manifestPath)) return null; + const src = fs.readFileSync(manifestPath, 'utf8'); + const lines = src.split(/\r?\n/); + const sections = []; + let currentSection = null; + let inTable = false; + for (const line of lines) { + const headerMatch = line.match(/^##\s+(.+)$/); + if (headerMatch) { + currentSection = { title: headerMatch[1].trim(), keys: [] }; + sections.push(currentSection); + inTable = false; + continue; + } + if (!currentSection) continue; + if (/^\|\s*(Key|Topic|Field|#)\s*\|/i.test(line)) { inTable = true; continue; } + if (inTable && /^\|[\s-]+\|/.test(line)) continue; + if (inTable && !line.trim().startsWith('|')) { + if (line.trim().startsWith('### ')) continue; + inTable = false; + continue; + } + if (!inTable) continue; + const cells = line.split('|').slice(1, -1).map((c) => c.trim()); + if (cells.length === 0) continue; + const keyCell = cells[0]; + const keyMatch = keyCell.match(/`([^`]+)`/); + if (keyMatch) currentSection.keys.push(keyMatch[1]); + } + return sections; +} + +function findTestFiles(nodeDir) { + const out = []; + function walk(dir) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.isFile() && /\.test\.js$/.test(entry.name)) out.push(full); + } + } + walk(path.join(nodeDir, 'test')); + return out; +} + +function keyIsCovered(key, testSources) { + if (!key) return true; + if (/^[a-z]+$/i.test(key) && key.length <= 4) return true; + const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const pattern = new RegExp(`['"\\\`]${escaped}['"\\\`]|\\.${escaped}\\b|\\b${escaped}\\b`); + return testSources.some((src) => pattern.test(src)); +} + +function verifyNode(nodeDir, opts) { + const nodeName = path.basename(nodeDir); + const manifestPath = path.join(nodeDir, 'test/_output-manifest.md'); + const findings = []; + const sections = parseManifest(manifestPath); + if (sections === null) { + findings.push({ severity: opts.strict ? 'error' : 'warn', msg: 'test/_output-manifest.md does not exist (output-coverage rule §3 requires it).' }); + return { node: nodeName, findings, manifest: null }; + } + if (sections.length === 0) { + findings.push({ severity: 'error', msg: 'test/_output-manifest.md has no sections (## headings) — empty or malformed.' }); + return { node: nodeName, findings, manifest: sections }; + } + const totalKeys = sections.reduce((s, sec) => s + sec.keys.length, 0); + if (totalKeys === 0) { + findings.push({ severity: 'error', msg: 'No keys parsed from any section table — check that ` `tickmarks wrap each Key/Topic/Field cell.' }); + return { node: nodeName, findings, manifest: sections }; + } + const testFiles = findTestFiles(nodeDir); + const testSources = testFiles.map((f) => fs.readFileSync(f, 'utf8')); + if (testFiles.length === 0) { + findings.push({ severity: 'error', msg: 'manifest declares keys but no test/**/*.test.js files exist.' }); + return { node: nodeName, findings, manifest: sections }; + } + for (const section of sections) { + for (const key of section.keys) { + if (!keyIsCovered(key, testSources)) { + findings.push({ + severity: 'warn', + section: section.title, + key, + msg: `key \`${key}\` declared in "${section.title}" but no test file references it.`, + }); + } + } + } + return { node: nodeName, findings, manifest: sections, totalKeys }; +} + +function findNodes(repoRoot) { + const nodesDir = path.join(repoRoot, 'nodes'); + return fs.readdirSync(nodesDir) + .filter((n) => fs.existsSync(path.join(nodesDir, n, 'CONTRACT.md'))) + .filter((n) => n !== 'generalFunctions') + .map((n) => path.join(nodesDir, n)); +} + +function report(result, json) { + if (json) { + process.stdout.write(JSON.stringify(result) + '\n'); + return result.findings.some((f) => f.severity === 'error'); + } + const errs = result.findings.filter((f) => f.severity === 'error').length; + const warns = result.findings.filter((f) => f.severity === 'warn').length; + if (errs === 0 && warns === 0) { + process.stdout.write(`OK ${result.node} (manifest covers ${result.totalKeys} keys)\n`); + return false; + } + const tag = errs ? 'FAIL' : 'WARN'; + process.stdout.write(`\n${tag} ${result.node} (${errs} err, ${warns} warn)\n`); + for (const f of result.findings) { + const t = f.severity === 'error' ? 'ERR ' : 'WARN'; + process.stdout.write(` ${t} ${f.msg}\n`); + } + return errs > 0; +} + +function main() { + const args = process.argv.slice(2); + const json = args.includes('--json'); + const strict = args.includes('--strict'); + const positional = args.filter((a) => !a.startsWith('--')); + const repoRoot = path.resolve(__dirname, '../../..'); + const targets = positional.length === 0 + ? findNodes(repoRoot) + : positional.map((p) => path.resolve(p)); + let fail = false; + for (const nodeDir of targets) { + const result = verifyNode(nodeDir, { strict }); + if (report(result, json)) fail = true; + } + process.exit(fail ? 1 : 0); +} + +if (require.main === module) main(); + +module.exports = { parseManifest, verifyNode, keyIsCovered }; diff --git a/tools/output-manifest-verify/package.json b/tools/output-manifest-verify/package.json new file mode 100644 index 0000000..abe8b7a --- /dev/null +++ b/tools/output-manifest-verify/package.json @@ -0,0 +1,13 @@ +{ + "name": "@evolv/output-manifest-verify", + "version": "0.1.0", + "private": true, + "description": "Verify each node ships test/_output-manifest.md and that declared keys appear in test files (per .claude/rules/output-coverage.md)", + "bin": { + "evolv-output-manifest-verify": "bin/output-manifest-verify.js" + }, + "scripts": { + "test": "node --test test/*.test.js" + }, + "license": "UNLICENSED" +}