tools: add output-manifest-verify; extend flow-lint with fan-out checks
- tools/output-manifest-verify/ — enforces .claude/rules/output-coverage.md
§3: every node ships test/_output-manifest.md and every declared key
is referenced by at least one test file. First run shows only
machineGroupControl has the manifest (16 keys covered); all other nodes
warn. --strict escalates "missing manifest" to an error for CI gating.
- flow-lint gains two rules from the same output-coverage rule:
* FN_OUTPUT_WIRES_MISMATCH — function declares outputs=N but wires has
M arrays (causes silent dropped or duplicate emissions).
* FN_PAYLOAD_NULL_LITERAL — function source contains `payload: null`
literal (the η-null ui-chart crash pattern from 2026-05-14).
First run found 1 instance in mgc/02-Dashboard.json.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -35,6 +35,7 @@ function lintFlow(flowPath) {
|
||||
checkLinkPair(n, byId, findings);
|
||||
checkDebugLog(n, findings);
|
||||
checkBackwardWires(n, findings);
|
||||
checkFunctionFanOut(n, findings);
|
||||
}
|
||||
checkGroupWidths(nodes, findings);
|
||||
return findings;
|
||||
@@ -172,6 +173,28 @@ function checkBackwardWires(n, findings) {
|
||||
}
|
||||
}
|
||||
|
||||
function checkFunctionFanOut(n, findings) {
|
||||
if (n.type !== 'function') return;
|
||||
const outputs = Number(n.outputs);
|
||||
if (!Number.isFinite(outputs) || outputs <= 1) return;
|
||||
if (!Array.isArray(n.wires) || n.wires.length !== outputs) {
|
||||
findings.push({
|
||||
rule: 'FN_OUTPUT_WIRES_MISMATCH',
|
||||
severity: 'error',
|
||||
node: n.id,
|
||||
msg: `function "${n.name || n.id}" declares outputs=${outputs} but wires has ${Array.isArray(n.wires) ? n.wires.length : 'no'} arrays.`,
|
||||
});
|
||||
}
|
||||
if (typeof n.func === 'string' && /payload\s*:\s*null\b/.test(n.func)) {
|
||||
findings.push({
|
||||
rule: 'FN_PAYLOAD_NULL_LITERAL',
|
||||
severity: 'error',
|
||||
node: n.id,
|
||||
msg: `function "${n.name || n.id}" emits a literal { payload: null } — crashes ui-chart on first frame; return the whole msg as null instead so the port skips.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function checkGroupWidths(nodes, findings) {
|
||||
const pages = new Map();
|
||||
for (const n of nodes) {
|
||||
|
||||
47
tools/output-manifest-verify/README.md
Normal file
47
tools/output-manifest-verify/README.md
Normal file
@@ -0,0 +1,47 @@
|
||||
# @evolv/output-manifest-verify
|
||||
|
||||
Enforce `.claude/rules/output-coverage.md` §3:
|
||||
|
||||
1. Every node ships `test/_output-manifest.md`.
|
||||
2. Every key declared in the manifest is referenced by at least one test
|
||||
file under `test/**/*.test.js`.
|
||||
|
||||
Designed to replace the manual verification checklist in the rule. The
|
||||
rule itself can become a thin pointer to this tool once every node is in
|
||||
compliance.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
# scan every node with a CONTRACT.md
|
||||
node tools/output-manifest-verify/bin/output-manifest-verify.js
|
||||
|
||||
# one node
|
||||
node tools/output-manifest-verify/bin/output-manifest-verify.js nodes/machineGroupControl
|
||||
|
||||
# fail (exit 1) when a manifest is missing, not just warn
|
||||
node tools/output-manifest-verify/bin/output-manifest-verify.js --strict
|
||||
|
||||
# CI / JSON output (one line per node)
|
||||
node tools/output-manifest-verify/bin/output-manifest-verify.js --json
|
||||
```
|
||||
|
||||
## Severity
|
||||
|
||||
| Condition | Severity |
|
||||
|---|---|
|
||||
| `test/_output-manifest.md` missing | `warn` (escalates to `error` under `--strict`) |
|
||||
| Manifest has no `##` sections / no tables | `error` |
|
||||
| Declared key not referenced by any test file | `warn` |
|
||||
| Manifest declares keys but `test/**/*.test.js` directory empty | `error` |
|
||||
|
||||
## What it does NOT check (yet)
|
||||
|
||||
- Whether each key has both a **populated** and **degraded** test (the
|
||||
rule's §3 promise). Reaching that level of confidence requires
|
||||
instrumenting the actual node runtime; tracked as a follow-up.
|
||||
- The Port 1 (InfluxDB) cardinality discipline — see
|
||||
`.claude/rules/telemetry.md`.
|
||||
|
||||
Run after touching `_output-manifest.md`, `test/**/output-*.test.js`,
|
||||
or any Port 0/1/2 emission key.
|
||||
148
tools/output-manifest-verify/bin/output-manifest-verify.js
Normal file
148
tools/output-manifest-verify/bin/output-manifest-verify.js
Normal file
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function parseManifest(manifestPath) {
|
||||
if (!fs.existsSync(manifestPath)) return null;
|
||||
const src = fs.readFileSync(manifestPath, 'utf8');
|
||||
const lines = src.split(/\r?\n/);
|
||||
const sections = [];
|
||||
let currentSection = null;
|
||||
let inTable = false;
|
||||
for (const line of lines) {
|
||||
const headerMatch = line.match(/^##\s+(.+)$/);
|
||||
if (headerMatch) {
|
||||
currentSection = { title: headerMatch[1].trim(), keys: [] };
|
||||
sections.push(currentSection);
|
||||
inTable = false;
|
||||
continue;
|
||||
}
|
||||
if (!currentSection) continue;
|
||||
if (/^\|\s*(Key|Topic|Field|#)\s*\|/i.test(line)) { inTable = true; continue; }
|
||||
if (inTable && /^\|[\s-]+\|/.test(line)) continue;
|
||||
if (inTable && !line.trim().startsWith('|')) {
|
||||
if (line.trim().startsWith('### ')) continue;
|
||||
inTable = false;
|
||||
continue;
|
||||
}
|
||||
if (!inTable) continue;
|
||||
const cells = line.split('|').slice(1, -1).map((c) => c.trim());
|
||||
if (cells.length === 0) continue;
|
||||
const keyCell = cells[0];
|
||||
const keyMatch = keyCell.match(/`([^`]+)`/);
|
||||
if (keyMatch) currentSection.keys.push(keyMatch[1]);
|
||||
}
|
||||
return sections;
|
||||
}
|
||||
|
||||
function findTestFiles(nodeDir) {
|
||||
const out = [];
|
||||
function walk(dir) {
|
||||
if (!fs.existsSync(dir)) return;
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(full);
|
||||
else if (entry.isFile() && /\.test\.js$/.test(entry.name)) out.push(full);
|
||||
}
|
||||
}
|
||||
walk(path.join(nodeDir, 'test'));
|
||||
return out;
|
||||
}
|
||||
|
||||
function keyIsCovered(key, testSources) {
|
||||
if (!key) return true;
|
||||
if (/^[a-z]+$/i.test(key) && key.length <= 4) return true;
|
||||
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const pattern = new RegExp(`['"\\\`]${escaped}['"\\\`]|\\.${escaped}\\b|\\b${escaped}\\b`);
|
||||
return testSources.some((src) => pattern.test(src));
|
||||
}
|
||||
|
||||
function verifyNode(nodeDir, opts) {
|
||||
const nodeName = path.basename(nodeDir);
|
||||
const manifestPath = path.join(nodeDir, 'test/_output-manifest.md');
|
||||
const findings = [];
|
||||
const sections = parseManifest(manifestPath);
|
||||
if (sections === null) {
|
||||
findings.push({ severity: opts.strict ? 'error' : 'warn', msg: 'test/_output-manifest.md does not exist (output-coverage rule §3 requires it).' });
|
||||
return { node: nodeName, findings, manifest: null };
|
||||
}
|
||||
if (sections.length === 0) {
|
||||
findings.push({ severity: 'error', msg: 'test/_output-manifest.md has no sections (## headings) — empty or malformed.' });
|
||||
return { node: nodeName, findings, manifest: sections };
|
||||
}
|
||||
const totalKeys = sections.reduce((s, sec) => s + sec.keys.length, 0);
|
||||
if (totalKeys === 0) {
|
||||
findings.push({ severity: 'error', msg: 'No keys parsed from any section table — check that ` `tickmarks wrap each Key/Topic/Field cell.' });
|
||||
return { node: nodeName, findings, manifest: sections };
|
||||
}
|
||||
const testFiles = findTestFiles(nodeDir);
|
||||
const testSources = testFiles.map((f) => fs.readFileSync(f, 'utf8'));
|
||||
if (testFiles.length === 0) {
|
||||
findings.push({ severity: 'error', msg: 'manifest declares keys but no test/**/*.test.js files exist.' });
|
||||
return { node: nodeName, findings, manifest: sections };
|
||||
}
|
||||
for (const section of sections) {
|
||||
for (const key of section.keys) {
|
||||
if (!keyIsCovered(key, testSources)) {
|
||||
findings.push({
|
||||
severity: 'warn',
|
||||
section: section.title,
|
||||
key,
|
||||
msg: `key \`${key}\` declared in "${section.title}" but no test file references it.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return { node: nodeName, findings, manifest: sections, totalKeys };
|
||||
}
|
||||
|
||||
function findNodes(repoRoot) {
|
||||
const nodesDir = path.join(repoRoot, 'nodes');
|
||||
return fs.readdirSync(nodesDir)
|
||||
.filter((n) => fs.existsSync(path.join(nodesDir, n, 'CONTRACT.md')))
|
||||
.filter((n) => n !== 'generalFunctions')
|
||||
.map((n) => path.join(nodesDir, n));
|
||||
}
|
||||
|
||||
function report(result, json) {
|
||||
if (json) {
|
||||
process.stdout.write(JSON.stringify(result) + '\n');
|
||||
return result.findings.some((f) => f.severity === 'error');
|
||||
}
|
||||
const errs = result.findings.filter((f) => f.severity === 'error').length;
|
||||
const warns = result.findings.filter((f) => f.severity === 'warn').length;
|
||||
if (errs === 0 && warns === 0) {
|
||||
process.stdout.write(`OK ${result.node} (manifest covers ${result.totalKeys} keys)\n`);
|
||||
return false;
|
||||
}
|
||||
const tag = errs ? 'FAIL' : 'WARN';
|
||||
process.stdout.write(`\n${tag} ${result.node} (${errs} err, ${warns} warn)\n`);
|
||||
for (const f of result.findings) {
|
||||
const t = f.severity === 'error' ? 'ERR ' : 'WARN';
|
||||
process.stdout.write(` ${t} ${f.msg}\n`);
|
||||
}
|
||||
return errs > 0;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const json = args.includes('--json');
|
||||
const strict = args.includes('--strict');
|
||||
const positional = args.filter((a) => !a.startsWith('--'));
|
||||
const repoRoot = path.resolve(__dirname, '../../..');
|
||||
const targets = positional.length === 0
|
||||
? findNodes(repoRoot)
|
||||
: positional.map((p) => path.resolve(p));
|
||||
let fail = false;
|
||||
for (const nodeDir of targets) {
|
||||
const result = verifyNode(nodeDir, { strict });
|
||||
if (report(result, json)) fail = true;
|
||||
}
|
||||
process.exit(fail ? 1 : 0);
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
|
||||
module.exports = { parseManifest, verifyNode, keyIsCovered };
|
||||
13
tools/output-manifest-verify/package.json
Normal file
13
tools/output-manifest-verify/package.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "@evolv/output-manifest-verify",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "Verify each node ships test/_output-manifest.md and that declared keys appear in test files (per .claude/rules/output-coverage.md)",
|
||||
"bin": {
|
||||
"evolv-output-manifest-verify": "bin/output-manifest-verify.js"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "node --test test/*.test.js"
|
||||
},
|
||||
"license": "UNLICENSED"
|
||||
}
|
||||
Reference in New Issue
Block a user