machineGroupControl/test/integration/idle-startup-deadlock.integration.test.js

// MGC + idle pumps under realistic startup times — three scenarios that
// pin down WHERE the live deadlock is happening when PS sends 100% but
// pumps "show on" without adopting the control value.
//
// All three scenarios start with idle pumps (NOT pre-started) and use
// non-zero state.time values so startup is observable. Each scenario
// prints the per-pump snapshot at the end. The asserts state what we
// EXPECT to happen — failures point at the exact codepath that breaks.
//
// Compare to demand-cycle-walkthrough.integration.test.js which
// pre-starts every pump to 'operational' and therefore CANNOT exercise
// the idle-during-rapid-retarget paths described here.

const test = require('node:test');
const assert = require('node:assert/strict');

const MachineGroup = require('../../src/specificClass');
const Machine = require('../../../rotatingMachine/src/specificClass');

const HEAD_MBAR_UP = 0;
const HEAD_MBAR_DOWN = 1100;
const N_PUMPS = 3;

const LOG_DEBUG = process.env.LOG_DEBUG === '1';
const logCfg = { enabled: LOG_DEBUG, logLevel: LOG_DEBUG ? 'debug' : 'error' };

// Production-realistic-but-shrunk: starting=1s, warmingup=2s. Total
// startup ~3s. Long enough for rapid retargeting (every 200ms) to land
// 10+ extra calls during the transient, short enough to keep the test
// well under 30s.
const stateConfig = {
  general: { logging: logCfg },
  state: { current: 'idle' },
  movement: { mode: 'staticspeed', speed: 200, maxSpeed: 200, interval: 50 },
  time: { starting: 1, warmingup: 2, stopping: 1, coolingdown: 2 },
};

function machineConfig(id) {
  return {
    general: { logging: logCfg, name: id, id, unit: 'm3/h' },
    functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' },
    asset: { category: 'pump', type: 'centrifugal', model: 'hidrostal-H05K-S03R', supplier: 'hidrostal' },
    mode: {
      current: 'auto',
      allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] },
      allowedSources: { auto: ['parent', 'GUI'] },
    },
    sequences: {
      startup:        ['starting', 'warmingup', 'operational'],
      shutdown:       ['stopping', 'coolingdown', 'idle'],
      emergencystop:  ['emergencystop', 'off'],
    },
  };
}

function groupConfig() {
  return {
    general: { logging: logCfg, name: 'mgc', id: 'mgc' },
    functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' },
    scaling: { current: 'normalized' },
    mode:    { current: 'optimalcontrol' },
  };
}

function buildGroup({ withPressure = true } = {}) {
  const mgc = new MachineGroup(groupConfig());
  const ids = Array.from({ length: N_PUMPS }, (_, i) => `pump_${String.fromCharCode(97 + i)}`);
  const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig));
  for (const m of pumps) {
    if (withPressure) {
      m.updateMeasuredPressure(HEAD_MBAR_UP, 'upstream', {
        timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` });
      m.updateMeasuredPressure(HEAD_MBAR_DOWN, 'downstream', {
        timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` });
    }
    mgc.childRegistrationUtils.registerChild(m, 'downstream');
  }
  mgc.calcAbsoluteTotals();
  mgc.calcDynamicTotals();
  return { mgc, pumps };
}

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

const NON_RUNNING = new Set(['idle', 'off', 'stopping', 'coolingdown', 'emergencystop']);
function snapshot(pump) {
  const state = pump.state.getCurrentState();
  const ctrl  = Number(pump.state.getCurrentPosition?.() ?? 0);
  const running = !NON_RUNNING.has(state);
  const flow  = running ? Number(pump.predictFlow?.outputY  ?? 0) * 3600 : 0;
  const power = running ? Number(pump.predictPower?.outputY ?? 0) / 1000 : 0;
  return { state, ctrl, flow, power, delayedMove: pump.state.delayedMove };
}

function printSnapshots(label, pumps) {
  console.log(`\n  --- ${label} ---`);
  console.log('  ' + ['id'.padEnd(8), 'state'.padEnd(14), 'ctrl%'.padStart(6), 'Q m³/h'.padStart(8), 'kW'.padStart(6), 'delayedMove'.padStart(12)].join(' '));
  console.log('  ' + '-'.repeat(60));
  for (const p of pumps) {
    const s = snapshot(p);
    console.log('  ' + [
      p.config.general.id.padEnd(8),
      s.state.padEnd(14),
      s.ctrl.toFixed(1).padStart(6),
      s.flow.toFixed(1).padStart(8),
      s.power.toFixed(1).padStart(6),
      String(s.delayedMove).padStart(12),
    ].join(' '));
  }
}

function expectAllRunningAt100(pumps, label) {
  // After settle every pump should be operational with high ctrl% and
  // measurable flow. "high" is conservative — at 100% normalized demand,
  // 3-pump split puts each pump near 100% ctrl. Allow >70% as the floor
  // (accommodates BEP-Gravitation's slight asymmetry at the curve edges).
  for (const p of pumps) {
    const s = snapshot(p);
    assert.equal(s.state, 'operational',
      `${label}: pump ${p.config.general.id} expected operational, got '${s.state}' (ctrl=${s.ctrl.toFixed(1)}, delayedMove=${s.delayedMove})`);
    assert.ok(s.ctrl > 70,
      `${label}: pump ${p.config.general.id} expected ctrl% > 70 at 100% demand, got ${s.ctrl.toFixed(2)} (state=${s.state}, delayedMove=${s.delayedMove})`);
    assert.ok(s.flow > 100,
      `${label}: pump ${p.config.general.id} expected flow > 100 m³/h, got ${s.flow.toFixed(2)} (state=${s.state}, ctrl=${s.ctrl.toFixed(1)})`);
  }
}

// ---------------------------------------------------------------------------
test('Scenario 1 — single-shot 100% demand to idle pumps', async () => {
  // Hypothesis A: a SINGLE handleInput call to MGC with all pumps idle is
  // enough to surface the bug. If pumps end up at 100% ctrl, the bug is
  // elsewhere (rapid retargeting OR pressure plumbing). If pumps stay at
  // 0%, the dispatch loop itself doesn't follow through on
  // execsequence-startup → flowmovement.

  const { mgc, pumps } = buildGroup();
  console.log(`\n[Scenario 1] head=${HEAD_MBAR_DOWN} mbar, time.starting=${stateConfig.time.starting}s, time.warmingup=${stateConfig.time.warmingup}s`);
  printSnapshots('before handleInput', pumps);

  await mgc.handleInput('parent', 100);
  printSnapshots('immediately after handleInput returns', pumps);

  // Wait for full startup (3s) + movement (~0.5s) + slack
  await sleep(6000);
  printSnapshots('after 6s settle', pumps);

  expectAllRunningAt100(pumps, 'Scenario 1');
});

// ---------------------------------------------------------------------------
test('Scenario 2 — rapid 100% retargeting during startup window', async () => {
  // Hypothesis B: PS fires _applyMachineGroupLevelControl on every level
  // tick (every few hundred ms). While pumps are in 'starting' /
  // 'warmingup', MGC's optimalControl loop snapshots them, hits NONE of
  // its three branches (idle / operational / flow<=0), and dispatches
  // nothing. The only reason pumps eventually move is the FIRST call's
  // queued `await flowmovement` after `await execsequence startup` —
  // unless a subsequent call's abortActiveMovements aborts that move
  // mid-flight, parking it in 'accelerating'/'decelerating'.

  const { mgc, pumps } = buildGroup();
  console.log(`\n[Scenario 2] firing mgc.handleInput('parent', 100) every 200ms for 5s`);
  printSnapshots('before any handleInput', pumps);

  // First call (kicks off startup); not awaited so retargets can layer on.
  mgc.handleInput('parent', 100).catch(e => console.log(`first call rejected: ${e.message}`));

  // Spam additional retargets every 200ms for 5s — covers the 3s startup
  // window with 25 extra retargeting calls.
  const interval = setInterval(() => {
    mgc.handleInput('parent', 100).catch(e => console.log(`retarget rejected: ${e.message}`));
  }, 200);
  await sleep(5000);
  clearInterval(interval);

  printSnapshots('right after retarget barrage stops', pumps);

  // Drain: let any pending moves finish and let the FSM settle.
  await sleep(3000);
  printSnapshots('after 3s drain', pumps);

  expectAllRunningAt100(pumps, 'Scenario 2');
});

// ---------------------------------------------------------------------------
test('Scenario 3 — pumps with NO pressure measurements injected', async () => {
  // Hypothesis C: in production, MGC may receive a demand BEFORE the
  // first pressure measurement has propagated. Without head, the curve's
  // operating point is at fDimension=defaults, and currentFxyYMin/Max
  // may not correspond to a usable envelope. If MGC's distributor then
  // hands every pump flow≤0, the dispatch loop falls into the 'flow<=0
  // → shutdown' branch and pumps go straight to idle.

  const { mgc, pumps } = buildGroup({ withPressure: false });
  const sample = pumps[0].groupPredictFlow ?? pumps[0].predictFlow;
  const minQ = sample.currentFxyYMin * 3600;
  const maxQ = sample.currentFxyYMax * 3600;
  const dyn = mgc.calcDynamicTotals();
  console.log(`\n[Scenario 3] no pressure injected. per-pump curve envelope: ${minQ.toFixed(1)} .. ${maxQ.toFixed(1)} m³/h, station: ${(dyn.flow.min*3600).toFixed(1)} .. ${(dyn.flow.max*3600).toFixed(1)} m³/h`);
  printSnapshots('before handleInput', pumps);

  await mgc.handleInput('parent', 100);
  await sleep(6000);
  printSnapshots('after 6s settle (no pressure)', pumps);

  // We don't assert success here — this scenario is exploratory. Just
  // log what happens. If pumps DO ramp despite no pressure, MGC is
  // resilient. If they stay idle, that's a meaningful failure mode for
  // the live system because a redeploy may rebuild the world before
  // sensors republish.
  console.log('  (Scenario 3 is exploratory — no asserts; review the snapshot above.)');
});

// ---------------------------------------------------------------------------
test('Scenario 4 — varying demand during startup (combo flips)', async () => {
  // Hypothesis D: in production the demand is NOT constant — as basin
  // level rises, percControl ramps from startLevel→maxLevel over the
  // basin model. Demand can flip between 1-pump / 2-pump / 3-pump
  // combinations every PS tick. Each flip in optimalControl tells some
  // pumps to start, others to shutdown, others nothing. If a pump that
  // was just told "startup" is told "shutdown" 1s later (still in
  // 'starting' state — neither idle nor operational), nothing happens
  // for that pump in this snapshot. The execsequence shutdown branch
  // requires state to be operational/accelerating/decelerating — a
  // 'starting'/'warmingup' pump is silently passed over for shutdown
  // too. The pump then proceeds to operational AND obeys its queued
  // flowmovement, even though MGC's intent has since changed.

  const { mgc, pumps } = buildGroup();
  const sequence = [25, 75, 50, 100, 30, 90, 60, 100];
  console.log(`\n[Scenario 4] varying demand sequence: ${sequence.join(' → ')} (each held 400ms)`);
  printSnapshots('before any handleInput', pumps);

  for (const pct of sequence) {
    console.log(`  → demand ${pct}%`);
    mgc.handleInput('parent', pct).catch(e => console.log(`call ${pct}% rejected: ${e.message}`));
    await sleep(400);
  }

  printSnapshots('right after sequence ends', pumps);

  // Final demand was 100% — drain and verify pumps converged.
  await sleep(4000);
  printSnapshots('after 4s drain (demand was last set to 100%)', pumps);

  expectAllRunningAt100(pumps, 'Scenario 4');
});