// Regression: pump A in pumpingstation-complete-example demo got stuck // running at minimum flow while basin level dropped past stopLevel and // kept dropping all the way to dry-run threshold. // // Root cause (two parts): // // 1. rotatingMachine.executeSequence on shutdown went through an // interruptible-abort path that returned the FSM to 'operational', // triggering state.transitionToState's auto-pickup of the queued // delayedMove — re-engaging the pump before the shutdown sequence // could reach stopping/coolingdown/idle. Fix: clear delayedMove at // the top of shutdown/emergencystop sequences. // // 2. PS calls turnOffAllMachines on every tick (every 2 s) while // level < stopLevel. Each call interrupted the still-running prior // shutdown's transitions, resetting the FSM to 'accelerating'. The // pump bounced accelerating ↔ decelerating forever and the actual // shutdown sequence transitions never ran. Fix: serialize per-pump // shutdown calls in turnOffAllMachines so concurrent invocations // are no-ops while a shutdown is already in flight. // // This test exercises part 2 — the per-pump serialization at the MGC // level — by hammering turnOffAllMachines from a tight loop, mirroring // the live tick cadence. const test = require('node:test'); const assert = require('node:assert/strict'); const MachineGroup = require('../../src/specificClass'); const Machine = require('../../../rotatingMachine/src/specificClass'); const logCfg = { enabled: false, logLevel: 'error' }; const stateConfig = { general: { logging: logCfg }, state: { current: 'idle' }, movement: { mode: 'staticspeed', speed: 50, maxSpeed: 100, interval: 10 }, // Non-zero shutdown timing so a shutdown takes long enough that a // concurrent turnOff call lands mid-sequence — exactly the live race. time: { starting: 0, warmingup: 0, stopping: 1, coolingdown: 1 }, }; function machineConfig(id) { return { general: { logging: logCfg, name: id, id, unit: 'm3/h' }, functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' }, asset: { model: 'hidrostal-H05K-S03R', unit: 'm3/h' }, mode: { current: 'auto', allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] }, allowedSources: { auto: ['parent', 'GUI'] }, }, sequences: { startup: ['starting', 'warmingup', 'operational'], shutdown: ['stopping', 'coolingdown', 'idle'], emergencystop: ['emergencystop', 'off'], }, }; } function groupConfig() { return { general: { logging: logCfg, name: 'mgc', id: 'mgc' }, functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' }, scaling: { current: 'normalized' }, mode: { current: 'optimalcontrol' }, }; } function buildGroup() { const mgc = new MachineGroup(groupConfig()); const ids = ['pump_a', 'pump_b', 'pump_c']; const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig)); for (const m of pumps) { m.updateMeasuredPressure(0, 'upstream', { timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` }); m.updateMeasuredPressure(1100, 'downstream', { timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` }); mgc.childRegistrationUtils.registerChild(m, 'downstream'); } mgc.calcAbsoluteTotals(); mgc.calcDynamicTotals(); return { mgc, pumps }; } const sleep = (ms) => new Promise(r => setTimeout(r, ms)); test('repeated turnOffAllMachines reaches idle (serializes concurrent shutdowns)', async () => { const { mgc, pumps } = buildGroup(); const pumpA = pumps[0]; // Start pump A and queue a delayedMove the way MGC's optimalControl // would when PS sends a 1% dead-zone keep-alive. await pumpA.handleInput('parent', 'execsequence', 'startup'); assert.equal(pumpA.state.getCurrentState(), 'operational'); pumpA.setpoint(80); // start a slow move (not awaited) await sleep(50); assert.equal(pumpA.state.getCurrentState(), 'accelerating'); pumpA.state.delayedMove = 75; // Mimic PS's tick loop: fire turnOffAllMachines on a tight cadence // without awaiting. Without the per-pump serialization in // turnOffAllMachines, each call hits the still-running prior shutdown // and bounces the pump back to accelerating — the live deadlock. const ticks = []; for (let i = 0; i < 6; i++) { ticks.push(mgc.turnOffAllMachines()); await sleep(80); // half the realtime tick — tighter race } await Promise.all(ticks); // Allow the (single) in-flight shutdown to finish its 1+1 s timed // transitions through stopping → coolingdown → idle. await sleep(2500); assert.equal(pumpA.state.getCurrentState(), 'idle', `pump must reach idle under repeated turnOff calls; got ${pumpA.state.getCurrentState()} (delayedMove=${pumpA.state.delayedMove})`); assert.equal(pumpA.state.delayedMove, null, 'delayedMove must be cleared after shutdown'); }); test('turnOffAllMachines cancels any parked demand so it cannot re-engage pumps', async () => { // PS sends a 1% keep-alive while MGC is mid-dispatch. MGC parks it in // its demand dispatcher's latest-wins slot. PS then crosses stopLevel // and calls turnOffAllMachines. Without cancelPending(), the parked // 1% call would fire AFTER the shutdown — re-engaging the pump. const { mgc } = buildGroup(); const gate = mgc._demandDispatcher._gate; // Pin a fake in-flight dispatch then park a pending call behind it. gate._inFlight = true; const parked = mgc.handleInput('parent', 1, Infinity, null); await mgc.turnOffAllMachines(); // Re-open the gate: the in-flight pin is artificial. Awaiting the // parked promise must yield the SUPERSEDED sentinel (i.e. it was // cancelled, not run). const res = await parked; assert.ok(res && res.superseded === true, 'parked demand must resolve as superseded after turnOffAllMachines cancels it'); // Idle now — pending slot must be clear. assert.equal(gate._pending, null, 'turnOff must cancel any parked demand so it cannot re-engage pumps post-shutdown'); gate._inFlight = false; });