8e684203a8
Scenario 5 covers 100% → 0% → 100% with the second 100% landing
mid-shutdown (stopping/coolingdown) — exercises the path where
delayedMove must NOT be saved on a non-idle non-residue state without
a follow-up startup, since transitionToState('idle') doesn't fire it.
Scenario 6 walks 10%→100%→10% monotonically and asserts the down-sweep's
final demand is honoured (catches the user's observed "stuck around
60% going up, no reaction going down" symptom — where pumps would
otherwise freeze at a stale setpoint from the up-sweep).
Both pass with the current MGC dispatch fix.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
355 lines
17 KiB
JavaScript
355 lines
17 KiB
JavaScript
// MGC + idle pumps under realistic startup times — three scenarios that
|
|
// pin down WHERE the live deadlock is happening when PS sends 100% but
|
|
// pumps "show on" without adopting the control value.
|
|
//
|
|
// All three scenarios start with idle pumps (NOT pre-started) and use
|
|
// non-zero state.time values so startup is observable. Each scenario
|
|
// prints the per-pump snapshot at the end. The asserts state what we
|
|
// EXPECT to happen — failures point at the exact codepath that breaks.
|
|
//
|
|
// Compare to demand-cycle-walkthrough.integration.test.js which
|
|
// pre-starts every pump to 'operational' and therefore CANNOT exercise
|
|
// the idle-during-rapid-retarget paths described here.
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
|
|
const MachineGroup = require('../../src/specificClass');
|
|
const Machine = require('../../../rotatingMachine/src/specificClass');
|
|
|
|
const HEAD_MBAR_UP = 0;
|
|
const HEAD_MBAR_DOWN = 1100;
|
|
const N_PUMPS = 3;
|
|
|
|
const LOG_DEBUG = process.env.LOG_DEBUG === '1';
|
|
const logCfg = { enabled: LOG_DEBUG, logLevel: LOG_DEBUG ? 'debug' : 'error' };
|
|
|
|
// Production-realistic-but-shrunk: starting=1s, warmingup=2s. Total
|
|
// startup ~3s. Long enough for rapid retargeting (every 200ms) to land
|
|
// 10+ extra calls during the transient, short enough to keep the test
|
|
// well under 30s.
|
|
const stateConfig = {
|
|
general: { logging: logCfg },
|
|
state: { current: 'idle' },
|
|
movement: { mode: 'staticspeed', speed: 200, maxSpeed: 200, interval: 50 },
|
|
time: { starting: 1, warmingup: 2, stopping: 1, coolingdown: 2 },
|
|
};
|
|
|
|
function machineConfig(id) {
|
|
return {
|
|
general: { logging: logCfg, name: id, id, unit: 'm3/h' },
|
|
functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' },
|
|
asset: { category: 'pump', type: 'centrifugal', model: 'hidrostal-H05K-S03R', supplier: 'hidrostal' },
|
|
mode: {
|
|
current: 'auto',
|
|
allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] },
|
|
allowedSources: { auto: ['parent', 'GUI'] },
|
|
},
|
|
sequences: {
|
|
startup: ['starting', 'warmingup', 'operational'],
|
|
shutdown: ['stopping', 'coolingdown', 'idle'],
|
|
emergencystop: ['emergencystop', 'off'],
|
|
},
|
|
};
|
|
}
|
|
|
|
function groupConfig() {
|
|
return {
|
|
general: { logging: logCfg, name: 'mgc', id: 'mgc' },
|
|
functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' },
|
|
scaling: { current: 'normalized' },
|
|
mode: { current: 'optimalcontrol' },
|
|
};
|
|
}
|
|
|
|
function buildGroup({ withPressure = true } = {}) {
|
|
const mgc = new MachineGroup(groupConfig());
|
|
const ids = Array.from({ length: N_PUMPS }, (_, i) => `pump_${String.fromCharCode(97 + i)}`);
|
|
const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig));
|
|
for (const m of pumps) {
|
|
if (withPressure) {
|
|
m.updateMeasuredPressure(HEAD_MBAR_UP, 'upstream', {
|
|
timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` });
|
|
m.updateMeasuredPressure(HEAD_MBAR_DOWN, 'downstream', {
|
|
timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` });
|
|
}
|
|
mgc.childRegistrationUtils.registerChild(m, 'downstream');
|
|
}
|
|
mgc.calcAbsoluteTotals();
|
|
mgc.calcDynamicTotals();
|
|
return { mgc, pumps };
|
|
}
|
|
|
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
|
|
const NON_RUNNING = new Set(['idle', 'off', 'stopping', 'coolingdown', 'emergencystop']);
|
|
function snapshot(pump) {
|
|
const state = pump.state.getCurrentState();
|
|
const ctrl = Number(pump.state.getCurrentPosition?.() ?? 0);
|
|
const running = !NON_RUNNING.has(state);
|
|
const flow = running ? Number(pump.predictFlow?.outputY ?? 0) * 3600 : 0;
|
|
const power = running ? Number(pump.predictPower?.outputY ?? 0) / 1000 : 0;
|
|
return { state, ctrl, flow, power, delayedMove: pump.state.delayedMove };
|
|
}
|
|
|
|
function printSnapshots(label, pumps) {
|
|
console.log(`\n --- ${label} ---`);
|
|
console.log(' ' + ['id'.padEnd(8), 'state'.padEnd(14), 'ctrl%'.padStart(6), 'Q m³/h'.padStart(8), 'kW'.padStart(6), 'delayedMove'.padStart(12)].join(' '));
|
|
console.log(' ' + '-'.repeat(60));
|
|
for (const p of pumps) {
|
|
const s = snapshot(p);
|
|
console.log(' ' + [
|
|
p.config.general.id.padEnd(8),
|
|
s.state.padEnd(14),
|
|
s.ctrl.toFixed(1).padStart(6),
|
|
s.flow.toFixed(1).padStart(8),
|
|
s.power.toFixed(1).padStart(6),
|
|
String(s.delayedMove).padStart(12),
|
|
].join(' '));
|
|
}
|
|
}
|
|
|
|
function expectAllRunningAt100(pumps, label) {
|
|
// After settle every pump should be operational with high ctrl% and
|
|
// measurable flow. "high" is conservative — at 100% normalized demand,
|
|
// 3-pump split puts each pump near 100% ctrl. Allow >70% as the floor
|
|
// (accommodates BEP-Gravitation's slight asymmetry at the curve edges).
|
|
for (const p of pumps) {
|
|
const s = snapshot(p);
|
|
assert.equal(s.state, 'operational',
|
|
`${label}: pump ${p.config.general.id} expected operational, got '${s.state}' (ctrl=${s.ctrl.toFixed(1)}, delayedMove=${s.delayedMove})`);
|
|
assert.ok(s.ctrl > 70,
|
|
`${label}: pump ${p.config.general.id} expected ctrl% > 70 at 100% demand, got ${s.ctrl.toFixed(2)} (state=${s.state}, delayedMove=${s.delayedMove})`);
|
|
assert.ok(s.flow > 100,
|
|
`${label}: pump ${p.config.general.id} expected flow > 100 m³/h, got ${s.flow.toFixed(2)} (state=${s.state}, ctrl=${s.ctrl.toFixed(1)})`);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
test('Scenario 1 — single-shot 100% demand to idle pumps', async () => {
|
|
// Hypothesis A: a SINGLE handleInput call to MGC with all pumps idle is
|
|
// enough to surface the bug. If pumps end up at 100% ctrl, the bug is
|
|
// elsewhere (rapid retargeting OR pressure plumbing). If pumps stay at
|
|
// 0%, the dispatch loop itself doesn't follow through on
|
|
// execsequence-startup → flowmovement.
|
|
|
|
const { mgc, pumps } = buildGroup();
|
|
console.log(`\n[Scenario 1] head=${HEAD_MBAR_DOWN} mbar, time.starting=${stateConfig.time.starting}s, time.warmingup=${stateConfig.time.warmingup}s`);
|
|
printSnapshots('before handleInput', pumps);
|
|
|
|
await mgc.handleInput('parent', 100);
|
|
printSnapshots('immediately after handleInput returns', pumps);
|
|
|
|
// Wait for full startup (3s) + movement (~0.5s) + slack
|
|
await sleep(6000);
|
|
printSnapshots('after 6s settle', pumps);
|
|
|
|
expectAllRunningAt100(pumps, 'Scenario 1');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
test('Scenario 2 — rapid 100% retargeting during startup window', async () => {
|
|
// Hypothesis B: PS fires _applyMachineGroupLevelControl on every level
|
|
// tick (every few hundred ms). While pumps are in 'starting' /
|
|
// 'warmingup', MGC's optimalControl loop snapshots them, hits NONE of
|
|
// its three branches (idle / operational / flow<=0), and dispatches
|
|
// nothing. The only reason pumps eventually move is the FIRST call's
|
|
// queued `await flowmovement` after `await execsequence startup` —
|
|
// unless a subsequent call's abortActiveMovements aborts that move
|
|
// mid-flight, parking it in 'accelerating'/'decelerating'.
|
|
|
|
const { mgc, pumps } = buildGroup();
|
|
console.log(`\n[Scenario 2] firing mgc.handleInput('parent', 100) every 200ms for 5s`);
|
|
printSnapshots('before any handleInput', pumps);
|
|
|
|
// First call (kicks off startup); not awaited so retargets can layer on.
|
|
mgc.handleInput('parent', 100).catch(e => console.log(`first call rejected: ${e.message}`));
|
|
|
|
// Spam additional retargets every 200ms for 5s — covers the 3s startup
|
|
// window with 25 extra retargeting calls.
|
|
const interval = setInterval(() => {
|
|
mgc.handleInput('parent', 100).catch(e => console.log(`retarget rejected: ${e.message}`));
|
|
}, 200);
|
|
await sleep(5000);
|
|
clearInterval(interval);
|
|
|
|
printSnapshots('right after retarget barrage stops', pumps);
|
|
|
|
// Drain: let any pending moves finish and let the FSM settle.
|
|
await sleep(3000);
|
|
printSnapshots('after 3s drain', pumps);
|
|
|
|
expectAllRunningAt100(pumps, 'Scenario 2');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
test('Scenario 3 — pumps with NO pressure measurements injected', async () => {
|
|
// Hypothesis C: in production, MGC may receive a demand BEFORE the
|
|
// first pressure measurement has propagated. Without head, the curve's
|
|
// operating point is at fDimension=defaults, and currentFxyYMin/Max
|
|
// may not correspond to a usable envelope. If MGC's distributor then
|
|
// hands every pump flow≤0, the dispatch loop falls into the 'flow<=0
|
|
// → shutdown' branch and pumps go straight to idle.
|
|
|
|
const { mgc, pumps } = buildGroup({ withPressure: false });
|
|
const sample = pumps[0].groupPredictFlow ?? pumps[0].predictFlow;
|
|
const minQ = sample.currentFxyYMin * 3600;
|
|
const maxQ = sample.currentFxyYMax * 3600;
|
|
const dyn = mgc.calcDynamicTotals();
|
|
console.log(`\n[Scenario 3] no pressure injected. per-pump curve envelope: ${minQ.toFixed(1)} .. ${maxQ.toFixed(1)} m³/h, station: ${(dyn.flow.min*3600).toFixed(1)} .. ${(dyn.flow.max*3600).toFixed(1)} m³/h`);
|
|
printSnapshots('before handleInput', pumps);
|
|
|
|
await mgc.handleInput('parent', 100);
|
|
await sleep(6000);
|
|
printSnapshots('after 6s settle (no pressure)', pumps);
|
|
|
|
// We don't assert success here — this scenario is exploratory. Just
|
|
// log what happens. If pumps DO ramp despite no pressure, MGC is
|
|
// resilient. If they stay idle, that's a meaningful failure mode for
|
|
// the live system because a redeploy may rebuild the world before
|
|
// sensors republish.
|
|
console.log(' (Scenario 3 is exploratory — no asserts; review the snapshot above.)');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
test('Scenario 5 — full up/down/up cycle through shutdown', async () => {
|
|
// Hypothesis E: when demand goes 100% → 0% → 100% (basin fills, drains
|
|
// past stopLevel, then refills), pumps pass through stopping →
|
|
// coolingdown → idle. If a fresh flow>0 demand arrives while a pump is
|
|
// mid-shutdown, the current MGC dispatch saves flowmovement to
|
|
// delayedMove (good) but doesn't issue execsequence-startup because
|
|
// state !== 'idle' (bug). The pump completes shutdown, reaches 'idle',
|
|
// and stays there because transitionToState('idle') doesn't fire
|
|
// delayedMove — only the transition INTO 'operational' does. Pump is
|
|
// stuck with delayedMove orphaned.
|
|
|
|
const { mgc, pumps } = buildGroup();
|
|
console.log('\n[Scenario 5] cycle: 100% → 0% → 100% with mid-shutdown re-engage');
|
|
printSnapshots('before any handleInput', pumps);
|
|
|
|
// Phase 1: drive up to 100% from idle.
|
|
await mgc.handleInput('parent', 100);
|
|
await sleep(5000); // full startup + ramp
|
|
printSnapshots('after settle at 100%', pumps);
|
|
for (const p of pumps) {
|
|
assert.equal(p.state.getCurrentState(), 'operational',
|
|
`Phase 1: pump ${p.config.general.id} not operational at 100% (got ${p.state.getCurrentState()})`);
|
|
}
|
|
|
|
// Phase 2: demand drops to 0% — pumps begin shutdown sequence.
|
|
// FIRE-AND-FORGET: handleInput(0) awaits turnOffAllMachines which
|
|
// awaits the full per-pump shutdown sequence. We need the next 100%
|
|
// demand to arrive WHILE pumps are still in stopping/coolingdown,
|
|
// not after they've reached idle.
|
|
mgc.handleInput('parent', 0).catch(e => console.log(`0% rejected: ${e.message}`));
|
|
// Wait briefly so the shutdown sequence enters but does NOT complete.
|
|
// shutdown=['stopping','coolingdown','idle'] with stopping=1s,
|
|
// coolingdown=2s. 500ms puts us solidly inside 'stopping'.
|
|
await sleep(500);
|
|
printSnapshots('mid-shutdown (pumps should be in stopping/coolingdown)', pumps);
|
|
|
|
const midShutdownStates = pumps.map(p => p.state.getCurrentState());
|
|
console.log(` states mid-shutdown: ${midShutdownStates.join(', ')}`);
|
|
|
|
// Phase 3: demand returns to 100% while pumps are mid-shutdown.
|
|
await mgc.handleInput('parent', 100);
|
|
// Generous: full coolingdown remaining + full startup + ramp.
|
|
await sleep(8000);
|
|
printSnapshots('after re-engage to 100%', pumps);
|
|
|
|
expectAllRunningAt100(pumps, 'Scenario 5');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
test('Scenario 6 — full up sweep then full down sweep', async () => {
|
|
// Hypothesis F: the user observed "going up stuck ~60%, going down
|
|
// not reacting". Mirror that with an explicit up-then-down monotonic
|
|
// sweep, every step holding 600 ms (slightly longer than DWELL on
|
|
// production basin model). After the sweep, we expect the LATEST
|
|
// demand (the final value of the down-sweep, which is 10%) to be
|
|
// honoured: pumps either at 1-pump combo's split or all idle if that
|
|
// demand falls below the per-pump minimum.
|
|
|
|
const { mgc, pumps } = buildGroup();
|
|
console.log('\n[Scenario 6] up-sweep 10%→100% then down-sweep 100%→10%, each step 600 ms');
|
|
printSnapshots('before any handleInput', pumps);
|
|
|
|
const upSteps = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
|
|
const downSteps = [90, 80, 70, 60, 50, 40, 30, 20, 10];
|
|
|
|
console.log(' --- up sweep ---');
|
|
for (const pct of upSteps) {
|
|
mgc.handleInput('parent', pct).catch(e => console.log(`up ${pct}% rejected: ${e.message}`));
|
|
await sleep(600);
|
|
const snaps = pumps.map(snapshot);
|
|
const totalQ = snaps.reduce((s, x) => s + x.flow, 0);
|
|
console.log(` cmd=${pct.toFixed(0).padStart(3)}% states=[${snaps.map(s=>s.state.padEnd(13)).join(', ')}] ctrl=[${snaps.map(s=>s.ctrl.toFixed(1).padStart(5)).join(', ')}] ΣQ=${totalQ.toFixed(1)}`);
|
|
}
|
|
printSnapshots('top of up-sweep (cmd=100%) after full settle', pumps);
|
|
await sleep(2000);
|
|
printSnapshots('top of up-sweep + 2s drain', pumps);
|
|
|
|
console.log(' --- down sweep ---');
|
|
for (const pct of downSteps) {
|
|
mgc.handleInput('parent', pct).catch(e => console.log(`down ${pct}% rejected: ${e.message}`));
|
|
await sleep(600);
|
|
const snaps = pumps.map(snapshot);
|
|
const totalQ = snaps.reduce((s, x) => s + x.flow, 0);
|
|
console.log(` cmd=${pct.toFixed(0).padStart(3)}% states=[${snaps.map(s=>s.state.padEnd(13)).join(', ')}] ctrl=[${snaps.map(s=>s.ctrl.toFixed(1).padStart(5)).join(', ')}] ΣQ=${totalQ.toFixed(1)}`);
|
|
}
|
|
printSnapshots('bottom of down-sweep (cmd=10%) after sequence', pumps);
|
|
await sleep(3000);
|
|
printSnapshots('bottom of down-sweep + 3s drain', pumps);
|
|
|
|
// Final demand was 10% (≈ 148 m³/h). At head 1100 mbar with per-pump
|
|
// min ≈ 89.5, this is solvable by a 1-pump combo near 148 m³/h.
|
|
// Optimizer typically picks the 1-pump combo. Either way, pumps are
|
|
// NOT supposed to be stuck at the prior up-sweep's 100% setpoint.
|
|
const flowMin_m3h = mgc.calcDynamicTotals().flow.min * 3600;
|
|
const flowMax_m3h = mgc.calcDynamicTotals().flow.max * 3600;
|
|
const expectedQ_m3h = flowMin_m3h + (flowMax_m3h - flowMin_m3h) * 0.10; // 10% scaled
|
|
console.log(` expected total flow at 10%: ~${expectedQ_m3h.toFixed(1)} m³/h`);
|
|
|
|
const snaps = pumps.map(snapshot);
|
|
const totalQ = snaps.reduce((s, x) => s + x.flow, 0);
|
|
// Loose: total within 30 m³/h of expectation. Catches the obvious
|
|
// stuck-at-old-position regression.
|
|
assert.ok(Math.abs(totalQ - expectedQ_m3h) < 30,
|
|
`Scenario 6: total flow ${totalQ.toFixed(1)} m³/h diverged from expected ${expectedQ_m3h.toFixed(1)} after down-sweep — pumps did not adopt latest demand. Per-pump: ${snaps.map(s => `${s.state}@${s.ctrl.toFixed(0)}%`).join(', ')}`);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
test('Scenario 4 — varying demand during startup (combo flips)', async () => {
|
|
// Hypothesis D: in production the demand is NOT constant — as basin
|
|
// level rises, percControl ramps from startLevel→maxLevel over the
|
|
// basin model. Demand can flip between 1-pump / 2-pump / 3-pump
|
|
// combinations every PS tick. Each flip in optimalControl tells some
|
|
// pumps to start, others to shutdown, others nothing. If a pump that
|
|
// was just told "startup" is told "shutdown" 1s later (still in
|
|
// 'starting' state — neither idle nor operational), nothing happens
|
|
// for that pump in this snapshot. The execsequence shutdown branch
|
|
// requires state to be operational/accelerating/decelerating — a
|
|
// 'starting'/'warmingup' pump is silently passed over for shutdown
|
|
// too. The pump then proceeds to operational AND obeys its queued
|
|
// flowmovement, even though MGC's intent has since changed.
|
|
|
|
const { mgc, pumps } = buildGroup();
|
|
const sequence = [25, 75, 50, 100, 30, 90, 60, 100];
|
|
console.log(`\n[Scenario 4] varying demand sequence: ${sequence.join(' → ')} (each held 400ms)`);
|
|
printSnapshots('before any handleInput', pumps);
|
|
|
|
for (const pct of sequence) {
|
|
console.log(` → demand ${pct}%`);
|
|
mgc.handleInput('parent', pct).catch(e => console.log(`call ${pct}% rejected: ${e.message}`));
|
|
await sleep(400);
|
|
}
|
|
|
|
printSnapshots('right after sequence ends', pumps);
|
|
|
|
// Final demand was 100% — drain and verify pumps converged.
|
|
await sleep(4000);
|
|
printSnapshots('after 4s drain (demand was last set to 100%)', pumps);
|
|
|
|
expectAllRunningAt100(pumps, 'Scenario 4');
|
|
});
|