machineGroupControl/test/integration/turnoff-deadlock.integration.test.js

// Regression: pump A in pumpingstation-complete-example demo got stuck
// running at minimum flow while basin level dropped past stopLevel and
// kept dropping all the way to dry-run threshold.
//
// Root cause (two parts):
//
// 1. rotatingMachine.executeSequence on shutdown went through an
//    interruptible-abort path that returned the FSM to 'operational',
//    triggering state.transitionToState's auto-pickup of the queued
//    delayedMove — re-engaging the pump before the shutdown sequence
//    could reach stopping/coolingdown/idle. Fix: clear delayedMove at
//    the top of shutdown/emergencystop sequences.
//
// 2. PS calls turnOffAllMachines on every tick (every 2 s) while
//    level < stopLevel. Each call interrupted the still-running prior
//    shutdown's transitions, resetting the FSM to 'accelerating'. The
//    pump bounced accelerating ↔ decelerating forever and the actual
//    shutdown sequence transitions never ran. Fix: serialize per-pump
//    shutdown calls in turnOffAllMachines so concurrent invocations
//    are no-ops while a shutdown is already in flight.
//
// This test exercises part 2 — the per-pump serialization at the MGC
// level — by hammering turnOffAllMachines from a tight loop, mirroring
// the live tick cadence.

const test = require('node:test');
const assert = require('node:assert/strict');

const MachineGroup = require('../../src/specificClass');
const Machine = require('../../../rotatingMachine/src/specificClass');

const logCfg = { enabled: false, logLevel: 'error' };

const stateConfig = {
  general: { logging: logCfg },
  state: { current: 'idle' },
  movement: { mode: 'staticspeed', speed: 50, maxSpeed: 100, interval: 10 },
  // Non-zero shutdown timing so a shutdown takes long enough that a
  // concurrent turnOff call lands mid-sequence — exactly the live race.
  time: { starting: 0, warmingup: 0, stopping: 1, coolingdown: 1 },
};

function machineConfig(id) {
  return {
    general: { logging: logCfg, name: id, id, unit: 'm3/h' },
    functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' },
    asset: { model: 'hidrostal-H05K-S03R', unit: 'm3/h' },
    mode: {
      current: 'auto',
      allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] },
      allowedSources: { auto: ['parent', 'GUI'] },
    },
    sequences: {
      startup:        ['starting', 'warmingup', 'operational'],
      shutdown:       ['stopping', 'coolingdown', 'idle'],
      emergencystop:  ['emergencystop', 'off'],
    },
  };
}

function groupConfig() {
  return {
    general: { logging: logCfg, name: 'mgc', id: 'mgc' },
    functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' },
    mode:    { current: 'optimalcontrol' },
  };
}

function buildGroup() {
  const mgc = new MachineGroup(groupConfig());
  const ids = ['pump_a', 'pump_b', 'pump_c'];
  const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig));
  for (const m of pumps) {
    m.updateMeasuredPressure(0,    'upstream',   { timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` });
    m.updateMeasuredPressure(1100, 'downstream', { timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` });
    mgc.childRegistrationUtils.registerChild(m, 'downstream');
  }
  mgc.calcAbsoluteTotals();
  mgc.calcDynamicTotals();
  return { mgc, pumps };
}

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

test('repeated turnOffAllMachines reaches idle (serializes concurrent shutdowns)', async () => {
  const { mgc, pumps } = buildGroup();
  const pumpA = pumps[0];

  // Start pump A and queue a delayedMove the way MGC's optimalControl
  // would when PS sends a 1% dead-zone keep-alive.
  await pumpA.handleInput('parent', 'execsequence', 'startup');
  assert.equal(pumpA.state.getCurrentState(), 'operational');
  pumpA.setpoint(80);                  // start a slow move (not awaited)
  await sleep(50);
  assert.equal(pumpA.state.getCurrentState(), 'accelerating');
  pumpA.state.delayedMove = 75;

  // Mimic PS's tick loop: fire turnOffAllMachines on a tight cadence
  // without awaiting. Without the per-pump serialization in
  // turnOffAllMachines, each call hits the still-running prior shutdown
  // and bounces the pump back to accelerating — the live deadlock.
  const ticks = [];
  for (let i = 0; i < 6; i++) {
    ticks.push(mgc.turnOffAllMachines());
    await sleep(80);                   // half the realtime tick — tighter race
  }
  await Promise.all(ticks);
  // Allow the (single) in-flight shutdown to finish its 1+1 s timed
  // transitions through stopping → coolingdown → idle.
  await sleep(2500);

  assert.equal(pumpA.state.getCurrentState(), 'idle',
    `pump must reach idle under repeated turnOff calls; got ${pumpA.state.getCurrentState()} (delayedMove=${pumpA.state.delayedMove})`);
  assert.equal(pumpA.state.delayedMove, null,
    'delayedMove must be cleared after shutdown');
});

test('turnOffAllMachines cancels any parked demand so it cannot re-engage pumps', async () => {
  // PS sends a 1% keep-alive while MGC is mid-dispatch. MGC parks it in
  // its demand dispatcher's latest-wins slot. PS then crosses stopLevel
  // and calls turnOffAllMachines. Without cancelPending(), the parked
  // 1% call would fire AFTER the shutdown — re-engaging the pump.
  const { mgc } = buildGroup();
  const gate = mgc._demandDispatcher._gate;
  // Pin a fake in-flight dispatch then park a pending call behind it.
  gate._inFlight = true;
  const parked = mgc.handleInput('parent', 1, Infinity, null);

  await mgc.turnOffAllMachines();

  // Re-open the gate: the in-flight pin is artificial. Awaiting the
  // parked promise must yield the SUPERSEDED sentinel (i.e. it was
  // cancelled, not run).
  const res = await parked;
  assert.ok(res && res.superseded === true,
    'parked demand must resolve as superseded after turnOffAllMachines cancels it');
  // Idle now — pending slot must be clear.
  assert.equal(gate._pending, null,
    'turnOff must cancel any parked demand so it cannot re-engage pumps post-shutdown');
  gate._inFlight = false;
});