app
Pup - the AI's own browser
Public Made by Adomby adom
pup gives your AI a real browser on your desktop that it drives completely, in the background, signed in as you: windows that never jump in front of your work, one shared profile that learns your logins, and every window labeled on the taskbar with its AI thread's name.
master
John Lauer
2.0.413: missing verbs + behaviors (assisted_click, import, fg-watch, url lint, notices, chrome-stable, eval frame, mov…
0dd55ee
2mo ago
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768
//! pup_assisted_click (Node: the six-step native click protocol, John's hard rule of 2026-07-29):
//! caption + toast first, raise the window, WAIT until the user is actually on it (foreground stable
//! for 1.2 s), re-check right before acting, then a real-cursor hover and click through ab aimed by
//! a fresh window screenshot, narrating every step. Never dispatches while the user is elsewhere.
use crate::{resolve, touch, Ctx};
use pup_core::{blocked_on_user, err, ok, Caller};
use serde_json::{json, Value};
use std::time::{Duration, Instant};
const AB_T: Duration = Duration::from_secs(8);
fn caption(ctx: &Ctx, caller: &Caller, text: &str, ms: u64) {
if ctx.settings.get("fgCaption", "on") == "off" { return; }
let _ = ctx.ab.call("desktop_caption", json!({ "id": "pup-assist", "text": text, "duration": ms, "expiresInMs": ms, "position": "bottom", "size": "medium" }), caller, AB_T);
}
/// Poll the OS foreground window until it is `hwnd` and stays there 1.2 s (at most 3 reminder captions).
pub fn wait_for_user_ready(ctx: &Ctx, caller: &Caller, hwnd: i64, max: Duration) -> bool {
let t0 = Instant::now(); let mut stable: Option<Instant> = None; let mut caps = 0; let mut last_cap = Instant::now() - Duration::from_secs(10);
let cap_ms: u64 = ctx.settings.get("fgCaptionMs", "3500").parse::<u64>().unwrap_or(2000).clamp(500, 12000);
while t0.elapsed() < max {
if ctx.host.foreground() == hwnd {
match stable { Some(s) if s.elapsed() >= Duration::from_millis(1200) => return true, Some(_) => {}, None => stable = Some(Instant::now()) }
} else {
stable = None;
if caps < 3 && last_cap.elapsed() > Duration::from_millis(3500) { caps += 1; last_cap = Instant::now(); caption(ctx, caller, "Adom is waiting: click back into THIS window when you are ready, and I will continue. (Nothing happens until you do.)", cap_ms); }
}
std::thread::sleep(Duration::from_millis(400));
}
false
}
pub fn assisted_click(ctx: &Ctx, caller: &Caller, args: &Value) -> Value {
let s = match resolve(ctx, caller, args) { Ok(s) => s, Err(e) => return e };
let (Some(x), Some(y)) = (args["x"].as_f64(), args["y"].as_f64()) else { return err("args_required", "pup_assisted_click needs window-image x+y (from a desktop_screenshot_window of this session) plus reason+label.", None) };
if !cfg!(windows) { return err("unsupported_platform", "assisted click is Windows-only (uses the OS cursor + foreground detection).", None); }
let reason: String = args["reason"].as_str().unwrap_or("Adom needs to click something on your screen.").chars().take(180).collect();
let label: String = args["label"].as_str().unwrap_or("the highlighted control").chars().take(80).collect();
caption(ctx, caller, &format!("Adom: {reason} I will bring this window up and WAIT until you are ready before clicking {label}."), 9000);
let _ = ctx.ab.call("notify_user", json!({ "title": "Adom needs one click", "body": format!("{label} — {reason} I will wait until you are on the window."), "focus": false }), caller, Duration::from_secs(4));
ctx.sessions.with_mut(&s.id, |x| { x.foreground = true; x.last_foreground = Some(json!({ "ts": pup_core::now_ms(), "verb": "pup_assisted_click", "reason": reason })); });
// Bring it to the user's screen directly (the caption + toast above are the narration; raise() would add its own).
let Some(hwnd) = crate::window::hwnd_of_pub(ctx, &s) else { return err("no_window_handle", "pup has no OS window handle for this window", Some("pup_rescan {} re-resolves handles.")) };
if ctx.ab.call("desktop_bring_to_front", json!({ "hwnd": hwnd }), caller, AB_T).is_err() { let _ = ctx.ab.call("desktop_window_show", json!({ "hwnd": hwnd }), caller, AB_T); }
let max_wait = Duration::from_millis(args["maxWaitMs"].as_u64().unwrap_or(120_000).min(45_000));
if !wait_for_user_ready(ctx, caller, hwnd, max_wait) {
caption(ctx, caller, "Adom is still waiting for you to come back to this window; your AI will ask again when you are ready.", 7000);
return blocked_on_user(false, "The user must bring the pup window to the foreground (click it or Alt-Tab to it) so the real-cursor click lands in it and nowhere else.", "Nothing was clicked. Tell the user which window to click into and call pup_assisted_click again with the same args; pup waits up to 45 s each call and never dispatches while the user is elsewhere.", json!({ "reason": "user_not_ready", "sessionId": s.id, "window": s.id, "hwnd": hwnd }));
}
caption(ctx, caller, &format!("Adom is clicking {label} now."), 5000);
let shot = ctx.ab.call("desktop_screenshot_window", json!({ "hwnd": hwnd }), caller, Duration::from_secs(8)).unwrap_or(Value::Null);
let inner = shot.get("output").and_then(|v| v.as_str()).and_then(|o| serde_json::from_str::<Value>(o).ok());
let pick = |v: &Value| v.get("shotId").and_then(|x| x.as_str()).map(|s| s.to_string()).or_else(|| v["screenshots"][0]["shotId"].as_str().map(|s| s.to_string()));
let shot_id = pick(&shot).or_else(|| inner.as_ref().and_then(pick));
let Some(shot_id) = shot_id else { return err("aim_failed", "could not capture the window to aim the click", Some("desktop_screenshot_window answered without a shotId; retry once, then fall back to pup_click if the control is in-page.")) };
// re-verify RIGHT before each native event: focus can move between two events
if ctx.host.foreground() != hwnd { return blocked_on_user(false, "The user left the window between the wait and the click.", "Nothing was clicked (the foreground moved). Call pup_assisted_click again.", json!({ "reason": "focus_moved", "sessionId": s.id })); }
let (xi, yi) = (x.round() as i64, y.round() as i64);
let _ = ctx.ab.call("desktop_hover", json!({ "space": "image", "shotId": shot_id, "x": xi, "y": yi }), caller, Duration::from_secs(6));
std::thread::sleep(Duration::from_millis(900));
if ctx.host.foreground() != hwnd { return blocked_on_user(false, "The user left the window right before the click.", "Nothing was clicked (the foreground moved after the hover). Call pup_assisted_click again.", json!({ "reason": "focus_moved", "sessionId": s.id })); }
let clicked = ctx.ab.call("desktop_click", json!({ "space": "image", "shotId": shot_id, "x": xi, "y": yi }), caller, Duration::from_secs(6)).unwrap_or(Value::Null);
caption(ctx, caller, &format!("Adom clicked {label}. Continuing."), 5000);
touch(ctx, &s.id, Some(&format!("assisted click on {label}")));
ok(json!({ "success": true, "output": json!({ "ok": true, "clicked": label, "x": xi, "y": yi, "sessionId": s.id, "abClick": clicked.get("success").cloned().unwrap_or(Value::Null) }).to_string(), "clicked": label, "x": xi, "y": yi, "sessionId": s.id, "window": s.id, "aiThread": s.owner,
"_hint": "Assisted click done after the user was confirmed on the window. Verify the effect with pup_eval or a screenshot; the window stays in the foreground (the user is on it)." }))
}