← Commit history

The hang gate never kills a Fusion that is using CPU; busy replies say what kept it alive (keptBy)

Drew Owens ·d8e2830b74 ·13d ago ·parent bd6f4e7
3 files changed +40−6
rust/bridge/src/launch.rs+2−2
@@ -351,8 +351,8 @@ pub fn gate(ctx: &Ctx, cmd: &str, args: &Value) -> Option<Value> {     }     if matches!(h["phase"].as_str(), Some("suspect") | Some("busy-not-hung") | Some("long-op-in-flight")) {         return Some(json!({"success": false, "error": "Fusion's main thread is not answering right now - most likely WORKING on something heavy (a STEP import or save starves the add-in's probes), not hung.",-            "errorCode": "fusion_addin_busy", "hostResponding": h["phase"].as_str() == Some("busy-not-hung"), "hungHeldSeconds": h["heldSeconds"],-            "_hint": "WAIT and re-poll; do not restart anything. The bridge only declares a real hang after 180s of continuous silence including a 20s probe - if that happens it restarts Fusion itself and says so."}));+            "errorCode": "fusion_addin_busy", "hostResponding": h["phase"].as_str() == Some("busy-not-hung"), "hungHeldSeconds": h["heldSeconds"], "keptBy": h["keptBy"],+            "_hint": "WAIT and re-poll; do not restart anything. The bridge only declares a real hang after 180s of continuous silence with Fusion idle (not using CPU), including a 20s probe - if that happens it restarts Fusion itself and says so."}));     }     // A modal in front of the add-in is never a reason to restart (issue #87, John: "screenshot     // itself using ab ... give the ai the parent/child array of screenshots so the ai can figure
rust/bridge/src/lifecycle.rs+2−2
@@ -529,8 +529,8 @@ pub fn diagnose_addin_timeout(ctx: &Ctx, command: &str, original: &Value) -> Val     }     let hung = crate::readiness_exec::confirmed_hung(ctx);     if matches!(hung["phase"].as_str(), Some("suspect") | Some("busy-not-hung") | Some("long-op-in-flight")) {-        return json!({"success": false, "error": "Fusion's main thread is not answering right now - most likely WORKING on something heavy, not hung (the probes it starves are the very ones this gate reads).", "errorCode": "fusion_addin_busy", "hostResponding": hung["phase"].as_str() == Some("busy-not-hung"), "hungHeldSeconds": hung["heldSeconds"],-            "_hint": "WAIT and re-poll. A real hang is only declared after 180s of continuous silence including a 20s probe, and then the bridge restarts Fusion itself."});+        return json!({"success": false, "error": "Fusion's main thread is not answering right now - most likely WORKING on something heavy, not hung (the probes it starves are the very ones this gate reads).", "errorCode": "fusion_addin_busy", "hostResponding": hung["phase"].as_str() == Some("busy-not-hung"), "hungHeldSeconds": hung["heldSeconds"], "keptBy": hung["keptBy"],+            "_hint": "WAIT and re-poll. A real hang is only declared after 180s of continuous silence with Fusion idle (not using CPU), including a 20s probe, and then the bridge restarts Fusion itself."});     }     if hung["hung"].as_bool() == Some(true) {         let _ = kill(ctx, &json!({})); std::thread::sleep(Duration::from_secs(6)); let _ = crate::launch::start(ctx, &json!({}));
rust/bridge/src/readiness_exec.rs+36−2
@@ -150,8 +150,24 @@ pub fn fusion_using_cpu(ctx: &Ctx) -> bool {     t1.saturating_sub(t0) > 50_000 } -/// `_confirmed_hung`: kill-grade only when the OS says hung AND a 20s probe gets nothing AND it-/// has persisted 180s; a long op in flight (under 30 minutes, or with CPU still moving) protects.+/// CPU over a sample window that says Fusion is working: at least 15% of one core across the family+/// (GetProcessTimes counts 100 ns units). A hung host's background threads stay well under it.+pub fn working_from_ticks(delta: u64, window: Duration) -> bool {+    delta as f64 >= 0.15 * window.as_secs_f64() * 10_000_000.0+}++/// A Fusion still using CPU is working, whatever its window says: a script or native call that+/// outlived the request that started it burns a core on the main thread, where a real hang (a+/// deadlock, a wait on a lost modal) sits near idle.+pub fn fusion_working(ctx: &Ctx) -> bool {+    let window = Duration::from_millis(1500);+    let t0 = ctx.host.family_cpu_ticks(); std::thread::sleep(window); let t1 = ctx.host.family_cpu_ticks();+    working_from_ticks(t1.saturating_sub(t0), window)+}++/// `_confirmed_hung`: kill-grade only when the OS says hung AND a 20s probe gets nothing AND the host+/// is idle AND all of that has persisted 180s; a long op in flight (under 30 minutes, or with CPU+/// still moving) protects. pub fn confirmed_hung(ctx: &Ctx) -> Value {     if let Some(lo) = crate::long_op_marker() {         let el = now_f() - lo.1;@@ -161,6 +177,10 @@ pub fn confirmed_hung(ctx: &Ctx) -> Value {     let h = is_fusion_hung(ctx);     if h["hung"].as_bool() != Some(true) { *HUNG_SINCE.lock().unwrap() = 0.0; return json!({"hung": false, "phase": "pumping"}); }     if crate::addin::addin_status_probe(Duration::from_secs(20)).is_some() { *HUNG_SINCE.lock().unwrap() = 0.0; return json!({"hung": false, "phase": "busy-not-hung", "signals": h["signals"]}); }+    // 2026-10-30: the gate killed a Fusion 37 minutes into a long build script, after the request+    // that started the script had timed out (so no long op was on record), and closed the user's+    // 24 documents. A kill now also needs the host to have gone idle.+    if fusion_working(ctx) { *HUNG_SINCE.lock().unwrap() = 0.0; return json!({"hung": false, "phase": "busy-not-hung", "keptBy": "fusion-cpu-active", "signals": h["signals"]}); }     let now = now_f();     let mut since = HUNG_SINCE.lock().unwrap();     if *since == 0.0 { *since = now; }@@ -168,3 +188,17 @@ pub fn confirmed_hung(ctx: &Ctx) -> Value {     if held < 180.0 { return json!({"hung": false, "phase": "suspect", "heldSeconds": (held * 10.0).round() / 10.0, "signals": h["signals"]}); }     json!({"hung": true, "phase": "confirmed", "heldSeconds": (held * 10.0).round() / 10.0, "signals": h["signals"]}) }++#[cfg(test)]+mod tests {+    use super::*;++    #[test]+    fn a_fusion_burning_a_core_is_working_not_hung() {+        let window = Duration::from_millis(1500);+        assert!(working_from_ticks(15_000_000, window)); // a whole core: a script on the main thread+        assert!(working_from_ticks(2_250_000, window)); // 15%+        assert!(!working_from_ticks(300_000, window)); // 2%: a hung host's background threads+        assert!(!working_from_ticks(0, window));+    }+}