machineGroupControl/test/integration/idle-startup-deadlock.integration.test.js

// MGC + idle pumps under realistic startup times — three scenarios that
// pin down WHERE the live deadlock is happening when PS sends 100% but
// pumps "show on" without adopting the control value.
//
// All three scenarios start with idle pumps (NOT pre-started) and use
// non-zero state.time values so startup is observable. Each scenario
// prints the per-pump snapshot at the end. The asserts state what we
// EXPECT to happen — failures point at the exact codepath that breaks.
//
// Compare to demand-cycle-walkthrough.integration.test.js which
// pre-starts every pump to 'operational' and therefore CANNOT exercise
// the idle-during-rapid-retarget paths described here.

const test = require('node:test');
const assert = require('node:assert/strict');

const MachineGroup = require('../../src/specificClass');
const Machine = require('../../../rotatingMachine/src/specificClass');

const HEAD_MBAR_UP = 0;
const HEAD_MBAR_DOWN = 1100;
const N_PUMPS = 3;

const LOG_DEBUG = process.env.LOG_DEBUG === '1';
const logCfg = { enabled: LOG_DEBUG, logLevel: LOG_DEBUG ? 'debug' : 'error' };

// Production-realistic-but-shrunk: starting=1s, warmingup=2s. Total
// startup ~3s. Long enough for rapid retargeting (every 200ms) to land
// 10+ extra calls during the transient, short enough to keep the test
// well under 30s.
const stateConfig = {
  general: { logging: logCfg },
  state: { current: 'idle' },
  movement: { mode: 'staticspeed', speed: 200, maxSpeed: 200, interval: 50 },
  time: { starting: 1, warmingup: 2, stopping: 1, coolingdown: 2 },
};

function machineConfig(id) {
  return {
    general: { logging: logCfg, name: id, id, unit: 'm3/h' },
    functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' },
    asset: { model: 'hidrostal-H05K-S03R', unit: 'm3/h' },
    mode: {
      current: 'auto',
      allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] },
      allowedSources: { auto: ['parent', 'GUI'] },
    },
    sequences: {
      startup:        ['starting', 'warmingup', 'operational'],
      shutdown:       ['stopping', 'coolingdown', 'idle'],
      emergencystop:  ['emergencystop', 'off'],
    },
  };
}

function groupConfig() {
  return {
    general: { logging: logCfg, name: 'mgc', id: 'mgc' },
    functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' },
    mode:    { current: 'optimalcontrol' },
  };
}

// Post-refactor handleInput takes canonical m³/s. This helper mirrors what
// the set.demand handler does for a bare-number (percent) payload, so test
// scenarios that previously sent `mgc.handleInput('parent', pctToCanonical(mgc, 100))` (= 100 %)
// keep their intent.
function pctToCanonical(mgc, pct) {
  if (pct < 0) return -1;
  const dt = mgc.calcDynamicTotals();
  return mgc.interpolation.interpolate_lin_single_point(pct, 0, 100, dt.flow.min, dt.flow.max);
}

function buildGroup({ withPressure = true } = {}) {
  const mgc = new MachineGroup(groupConfig());
  const ids = Array.from({ length: N_PUMPS }, (_, i) => `pump_${String.fromCharCode(97 + i)}`);
  const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig));
  for (const m of pumps) {
    if (withPressure) {
      m.updateMeasuredPressure(HEAD_MBAR_UP, 'upstream', {
        timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` });
      m.updateMeasuredPressure(HEAD_MBAR_DOWN, 'downstream', {
        timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` });
    }
    mgc.childRegistrationUtils.registerChild(m, 'downstream');
  }
  mgc.calcAbsoluteTotals();
  mgc.calcDynamicTotals();
  return { mgc, pumps };
}

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

const NON_RUNNING = new Set(['idle', 'off', 'stopping', 'coolingdown', 'emergencystop']);
function snapshot(pump) {
  const state = pump.state.getCurrentState();
  const ctrl  = Number(pump.state.getCurrentPosition?.() ?? 0);
  const running = !NON_RUNNING.has(state);
  const flow  = running ? Number(pump.predictFlow?.outputY  ?? 0) * 3600 : 0;
  const power = running ? Number(pump.predictPower?.outputY ?? 0) / 1000 : 0;
  return { state, ctrl, flow, power, delayedMove: pump.state.delayedMove };
}

function printSnapshots(label, pumps) {
  console.log(`\n  --- ${label} ---`);
  console.log('  ' + ['id'.padEnd(8), 'state'.padEnd(14), 'ctrl%'.padStart(6), 'Q m³/h'.padStart(8), 'kW'.padStart(6), 'delayedMove'.padStart(12)].join(' '));
  console.log('  ' + '-'.repeat(60));
  for (const p of pumps) {
    const s = snapshot(p);
    console.log('  ' + [
      p.config.general.id.padEnd(8),
      s.state.padEnd(14),
      s.ctrl.toFixed(1).padStart(6),
      s.flow.toFixed(1).padStart(8),
      s.power.toFixed(1).padStart(6),
      String(s.delayedMove).padStart(12),
    ].join(' '));
  }
}

function expectAllRunningAt100(pumps, label) {
  // After settle every pump should be operational with high ctrl% and
  // measurable flow. "high" is conservative — at 100% normalized demand,
  // 3-pump split puts each pump near 100% ctrl. Allow >70% as the floor
  // (accommodates BEP-Gravitation's slight asymmetry at the curve edges).
  for (const p of pumps) {
    const s = snapshot(p);
    assert.equal(s.state, 'operational',
      `${label}: pump ${p.config.general.id} expected operational, got '${s.state}' (ctrl=${s.ctrl.toFixed(1)}, delayedMove=${s.delayedMove})`);
    assert.ok(s.ctrl > 70,
      `${label}: pump ${p.config.general.id} expected ctrl% > 70 at 100% demand, got ${s.ctrl.toFixed(2)} (state=${s.state}, delayedMove=${s.delayedMove})`);
    assert.ok(s.flow > 100,
      `${label}: pump ${p.config.general.id} expected flow > 100 m³/h, got ${s.flow.toFixed(2)} (state=${s.state}, ctrl=${s.ctrl.toFixed(1)})`);
  }
}

// ---------------------------------------------------------------------------
test('Scenario 1 — single-shot 100% demand to idle pumps', async () => {
  // Hypothesis A: a SINGLE handleInput call to MGC with all pumps idle is
  // enough to surface the bug. If pumps end up at 100% ctrl, the bug is
  // elsewhere (rapid retargeting OR pressure plumbing). If pumps stay at
  // 0%, the dispatch loop itself doesn't follow through on
  // execsequence-startup → flowmovement.

  const { mgc, pumps } = buildGroup();
  console.log(`\n[Scenario 1] head=${HEAD_MBAR_DOWN} mbar, time.starting=${stateConfig.time.starting}s, time.warmingup=${stateConfig.time.warmingup}s`);
  printSnapshots('before handleInput', pumps);

  await mgc.handleInput('parent', pctToCanonical(mgc, 100));
  printSnapshots('immediately after handleInput returns', pumps);

  // Wait for full startup (3s) + movement (~0.5s) + slack
  await sleep(6000);
  printSnapshots('after 6s settle', pumps);

  expectAllRunningAt100(pumps, 'Scenario 1');
});

// ---------------------------------------------------------------------------
test('Scenario 2 — rapid 100% retargeting during startup window', async () => {
  // Hypothesis B: PS fires _applyMachineGroupLevelControl on every level
  // tick (every few hundred ms). While pumps are in 'starting' /
  // 'warmingup', MGC's optimalControl loop snapshots them, hits NONE of
  // its three branches (idle / operational / flow<=0), and dispatches
  // nothing. The only reason pumps eventually move is the FIRST call's
  // queued `await flowmovement` after `await execsequence startup` —
  // unless a subsequent call's abortActiveMovements aborts that move
  // mid-flight, parking it in 'accelerating'/'decelerating'.

  const { mgc, pumps } = buildGroup();
  console.log(`\n[Scenario 2] firing mgc.handleInput('parent', pctToCanonical(mgc, 100)) every 200ms for 5s`);
  printSnapshots('before any handleInput', pumps);

  // First call (kicks off startup); not awaited so retargets can layer on.
  mgc.handleInput('parent', pctToCanonical(mgc, 100)).catch(e => console.log(`first call rejected: ${e.message}`));

  // Spam additional retargets every 200ms for 5s — covers the 3s startup
  // window with 25 extra retargeting calls.
  const interval = setInterval(() => {
    mgc.handleInput('parent', pctToCanonical(mgc, 100)).catch(e => console.log(`retarget rejected: ${e.message}`));
  }, 200);
  await sleep(5000);
  clearInterval(interval);

  printSnapshots('right after retarget barrage stops', pumps);

  // Drain: let any pending moves finish and let the FSM settle.
  await sleep(3000);
  printSnapshots('after 3s drain', pumps);

  expectAllRunningAt100(pumps, 'Scenario 2');
});

// ---------------------------------------------------------------------------
test('Scenario 3 — pumps with NO pressure measurements injected', async () => {
  // Hypothesis C: in production, MGC may receive a demand BEFORE the
  // first pressure measurement has propagated. Without head, the curve's
  // operating point is at fDimension=defaults, and currentFxyYMin/Max
  // may not correspond to a usable envelope. If MGC's distributor then
  // hands every pump flow≤0, the dispatch loop falls into the 'flow<=0
  // → shutdown' branch and pumps go straight to idle.

  const { mgc, pumps } = buildGroup({ withPressure: false });
  const sample = pumps[0].groupPredictFlow ?? pumps[0].predictFlow;
  const minQ = sample.currentFxyYMin * 3600;
  const maxQ = sample.currentFxyYMax * 3600;
  const dyn = mgc.calcDynamicTotals();
  console.log(`\n[Scenario 3] no pressure injected. per-pump curve envelope: ${minQ.toFixed(1)} .. ${maxQ.toFixed(1)} m³/h, station: ${(dyn.flow.min*3600).toFixed(1)} .. ${(dyn.flow.max*3600).toFixed(1)} m³/h`);
  printSnapshots('before handleInput', pumps);

  await mgc.handleInput('parent', pctToCanonical(mgc, 100));
  await sleep(6000);
  printSnapshots('after 6s settle (no pressure)', pumps);

  // We don't assert success here — this scenario is exploratory. Just
  // log what happens. If pumps DO ramp despite no pressure, MGC is
  // resilient. If they stay idle, that's a meaningful failure mode for
  // the live system because a redeploy may rebuild the world before
  // sensors republish.
  console.log('  (Scenario 3 is exploratory — no asserts; review the snapshot above.)');
});

// ---------------------------------------------------------------------------
test('Scenario 5 — full up/down/up cycle through shutdown', async () => {
  // Hypothesis E: when demand goes 100% → 0% → 100% (basin fills, drains
  // past stopLevel, then refills), pumps pass through stopping →
  // coolingdown → idle. If a fresh flow>0 demand arrives while a pump is
  // mid-shutdown, the current MGC dispatch saves flowmovement to
  // delayedMove (good) but doesn't issue execsequence-startup because
  // state !== 'idle' (bug). The pump completes shutdown, reaches 'idle',
  // and stays there because transitionToState('idle') doesn't fire
  // delayedMove — only the transition INTO 'operational' does. Pump is
  // stuck with delayedMove orphaned.

  const { mgc, pumps } = buildGroup();
  console.log('\n[Scenario 5] cycle: 100% → 0% → 100% with mid-shutdown re-engage');
  printSnapshots('before any handleInput', pumps);

  // Phase 1: drive up to 100% from idle.
  await mgc.handleInput('parent', pctToCanonical(mgc, 100));
  await sleep(5000); // full startup + ramp
  printSnapshots('after settle at 100%', pumps);
  for (const p of pumps) {
    assert.equal(p.state.getCurrentState(), 'operational',
      `Phase 1: pump ${p.config.general.id} not operational at 100% (got ${p.state.getCurrentState()})`);
  }

  // Phase 2: demand drops below 0 — pumps begin shutdown sequence. Use a
  // strictly-negative percent because 0% now means "minimum-control"
  // (interpolates to dt.flow.min), not shutdown.
  // FIRE-AND-FORGET: handleInput(-1) awaits turnOffAllMachines which
  // awaits the full per-pump shutdown sequence. We need the next 100%
  // demand to arrive WHILE pumps are still in stopping/coolingdown,
  // not after they've reached idle.
  mgc.turnOffAllMachines().catch(e => console.log(`-1% rejected: ${e.message}`));
  // Wait briefly so the shutdown sequence enters but does NOT complete.
  // shutdown=['stopping','coolingdown','idle'] with stopping=1s,
  // coolingdown=2s. 500ms puts us solidly inside 'stopping'.
  await sleep(500);
  printSnapshots('mid-shutdown (pumps should be in stopping/coolingdown)', pumps);

  const midShutdownStates = pumps.map(p => p.state.getCurrentState());
  console.log(`  states mid-shutdown: ${midShutdownStates.join(', ')}`);

  // Phase 3: demand returns to 100% while pumps are mid-shutdown.
  await mgc.handleInput('parent', pctToCanonical(mgc, 100));
  // Generous: full coolingdown remaining + full startup + ramp.
  await sleep(8000);
  printSnapshots('after re-engage to 100%', pumps);

  expectAllRunningAt100(pumps, 'Scenario 5');
});

// ---------------------------------------------------------------------------
test('Scenario 6 — full up sweep then full down sweep', async () => {
  // Hypothesis F: the user observed "going up stuck ~60%, going down
  // not reacting". Mirror that with an explicit up-then-down monotonic
  // sweep, every step holding 600 ms (slightly longer than DWELL on
  // production basin model). After the sweep, we expect the LATEST
  // demand (the final value of the down-sweep, which is 10%) to be
  // honoured: pumps either at 1-pump combo's split or all idle if that
  // demand falls below the per-pump minimum.

  const { mgc, pumps } = buildGroup();
  console.log('\n[Scenario 6] up-sweep 10%→100% then down-sweep 100%→10%, each step 600 ms');
  printSnapshots('before any handleInput', pumps);

  const upSteps   = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
  const downSteps = [90, 80, 70, 60, 50, 40, 30, 20, 10];

  console.log('  --- up sweep ---');
  for (const pct of upSteps) {
    mgc.handleInput('parent', pctToCanonical(mgc, pct)).catch(e => console.log(`up ${pct}% rejected: ${e.message}`));
    await sleep(600);
    const snaps = pumps.map(snapshot);
    const totalQ = snaps.reduce((s, x) => s + x.flow, 0);
    console.log(`    cmd=${pct.toFixed(0).padStart(3)}%  states=[${snaps.map(s=>s.state.padEnd(13)).join(', ')}] ctrl=[${snaps.map(s=>s.ctrl.toFixed(1).padStart(5)).join(', ')}] ΣQ=${totalQ.toFixed(1)}`);
  }
  printSnapshots('top of up-sweep (cmd=100%) after full settle', pumps);
  await sleep(2000);
  printSnapshots('top of up-sweep + 2s drain', pumps);

  console.log('  --- down sweep ---');
  for (const pct of downSteps) {
    mgc.handleInput('parent', pctToCanonical(mgc, pct)).catch(e => console.log(`down ${pct}% rejected: ${e.message}`));
    await sleep(600);
    const snaps = pumps.map(snapshot);
    const totalQ = snaps.reduce((s, x) => s + x.flow, 0);
    console.log(`    cmd=${pct.toFixed(0).padStart(3)}%  states=[${snaps.map(s=>s.state.padEnd(13)).join(', ')}] ctrl=[${snaps.map(s=>s.ctrl.toFixed(1).padStart(5)).join(', ')}] ΣQ=${totalQ.toFixed(1)}`);
  }
  printSnapshots('bottom of down-sweep (cmd=10%) after sequence', pumps);
  await sleep(3000);
  printSnapshots('bottom of down-sweep + 3s drain', pumps);

  // Final demand was 10% (≈ 148 m³/h). At head 1100 mbar with per-pump
  // min ≈ 89.5, this is solvable by a 1-pump combo near 148 m³/h.
  // Optimizer typically picks the 1-pump combo. Either way, pumps are
  // NOT supposed to be stuck at the prior up-sweep's 100% setpoint.
  const flowMin_m3h = mgc.calcDynamicTotals().flow.min * 3600;
  const flowMax_m3h = mgc.calcDynamicTotals().flow.max * 3600;
  const expectedQ_m3h = flowMin_m3h + (flowMax_m3h - flowMin_m3h) * 0.10; // 10% scaled
  console.log(`  expected total flow at 10%: ~${expectedQ_m3h.toFixed(1)} m³/h`);

  const snaps = pumps.map(snapshot);
  const totalQ = snaps.reduce((s, x) => s + x.flow, 0);
  // Loose: total within 30 m³/h of expectation. Catches the obvious
  // stuck-at-old-position regression.
  assert.ok(Math.abs(totalQ - expectedQ_m3h) < 30,
    `Scenario 6: total flow ${totalQ.toFixed(1)} m³/h diverged from expected ${expectedQ_m3h.toFixed(1)} after down-sweep — pumps did not adopt latest demand. Per-pump: ${snaps.map(s => `${s.state}@${s.ctrl.toFixed(0)}%`).join(', ')}`);
});

// ---------------------------------------------------------------------------
test('Scenario 4 — varying demand during startup (combo flips)', async () => {
  // Hypothesis D: in production the demand is NOT constant — as basin
  // level rises, percControl ramps from startLevel→maxLevel over the
  // basin model. Demand can flip between 1-pump / 2-pump / 3-pump
  // combinations every PS tick. Each flip in optimalControl tells some
  // pumps to start, others to shutdown, others nothing. If a pump that
  // was just told "startup" is told "shutdown" 1s later (still in
  // 'starting' state — neither idle nor operational), nothing happens
  // for that pump in this snapshot. The execsequence shutdown branch
  // requires state to be operational/accelerating/decelerating — a
  // 'starting'/'warmingup' pump is silently passed over for shutdown
  // too. The pump then proceeds to operational AND obeys its queued
  // flowmovement, even though MGC's intent has since changed.

  const { mgc, pumps } = buildGroup();
  const sequence = [25, 75, 50, 100, 30, 90, 60, 100];
  console.log(`\n[Scenario 4] varying demand sequence: ${sequence.join(' → ')} (each held 400ms)`);
  printSnapshots('before any handleInput', pumps);

  for (const pct of sequence) {
    console.log(`  → demand ${pct}%`);
    mgc.handleInput('parent', pctToCanonical(mgc, pct)).catch(e => console.log(`call ${pct}% rejected: ${e.message}`));
    await sleep(400);
  }

  printSnapshots('right after sequence ends', pumps);

  // Final demand was 100% — drain and verify pumps converged.
  await sleep(4000);
  printSnapshots('after 4s drain (demand was last set to 100%)', pumps);

  expectAllRunningAt100(pumps, 'Scenario 4');
});