git.lucas.co / cce-remote
remote trackpad and keyboard server
git clone https://git.lucas.co/cce-remote.git

commitf18ba7967bb7e5f53256a9ea018166ac7be5dbe1
parentaddb833dc8
authorLucas Galante <[email protected]>
date2026-07-21 20:25
feat: window view is a live MJPEG stream

/stream serves multipart/x-mixed-replace JPEG parts from grim region
captures of the focused window (half scale, q65; the region re-resolves
every few frames so the stream follows focus; loop ends when the client
closes). ~2.5 fps for a full-size window — the screencopy dominates.
The page renders it natively in the <img> (PIN via ?pin= query, since
img can't carry headers); winRect for tap mapping is kept fresh by
polling wl over the WS, and the switcher renders only on request so
those polls stay silent. Post-interaction refresh timers are gone —
the stream IS the feedback.

Co-Authored-By: Claude Fable 5 <[email protected]>

 README.md   | 10 ++++-----
 index.html  | 47 ++++++++++++++++++++---------------------
 src/main.rs | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 28 deletions(-)

diff --git a/README.md b/README.md
index 0af455a..d191b95 100644
--- a/README.md
+++ b/README.md
@@ -30,11 +30,11 @@ Home Screen for a fullscreen app feel.
   char→evdev map; iOS `beforeinput` is used, so autocorrect noise is
   filtered)
 - ☰ — window switcher: tap a window to focus it
-- 🖥 — window view mode: the pad shows the focused window (auto-refreshing
-  screenshot, ~1.5s + after each interaction). Tap to click that spot,
-  long-press to right-click, two-finger drag to scroll. Toggle again for
-  the trackpad. Screenshots are served PIN-gated and deleted immediately —
-  nothing accumulates on disk.
+- 🖥 — window view mode: a live MJPEG stream of the focused window
+  (~2-3 fps, grim region captures at half scale; the stream follows focus).
+  Tap to click that spot, long-press to right-click, two-finger drag to
+  scroll. Toggle again for the trackpad. Both `/stream` and the one-shot
+  `/shot` endpoint are PIN-gated; nothing accumulates on disk.
 
 ## Security
 
diff --git a/index.html b/index.html
index 5b7b4f9..9fce00f 100644
--- a/index.html
+++ b/index.html
@@ -124,24 +124,15 @@ const ACCEL = 1.6, SCROLL = 0.8;
 let touches = new Map(), moved = 0, startT = 0, maxFingers = 0, startX = 0, startY = 0;
 let dragging = false, dragTimer = null, pendDx = 0, pendDy = 0, pendSy = 0, flushT = null;
 
-// ── Window view mode: the pad shows the focused window; taps click it ──
-let viewMode = false, winRect = null, shotTimer = null, shotBusy = false;
+// ── Window view mode: live MJPEG stream of the focused window; taps
+// click it. The <img> renders /stream (multipart/x-mixed-replace)
+// natively; winRect for tap mapping is kept fresh by polling `wl` over
+// the WS (the stream follows focus server-side).
+let viewMode = false, winRect = null, rectTimer = null;
 const screenEl = document.getElementById("screen");
-async function refreshShot() {
-  if (!viewMode || shotBusy) return;
-  shotBusy = true;
-  try {
-    const r = await fetch("/shot", { headers: { "X-Pin": localStorage.getItem("cce-remote-pin") || "" }, cache: "no-store" });
-    if (r.ok) {
-      const xw = (r.headers.get("X-Win") || "").split(" ").map(Number);
-      if (xw.length === 5) winRect = { id: xw[0], x: xw[1], y: xw[2], w: xw[3], h: xw[4] };
-      const url = URL.createObjectURL(await r.blob());
-      const old = screenEl.src;
-      screenEl.onload = () => { if (old && old.startsWith("blob:")) URL.revokeObjectURL(old); };
-      screenEl.src = url;
-    }
-  } catch (_) {}
-  shotBusy = false;
+function updateRect(wins) {
+  const f = wins.find(w => w.focused && w.mode !== "Status");
+  if (f) winRect = { id: f.id, x: f.x, y: f.y, w: f.w, h: f.h };
 }
 // touch point (viewport px) → layout px inside the focused window,
 // accounting for the contain-fit letterbox. null when off the image.
@@ -180,7 +171,7 @@ pad.addEventListener("touchstart", e => {
       if (viewMode) {
         // long-press in view mode = right click at the spot
         const m = mapToWindow(startX, startY);
-        if (m) { send("tapr " + m[0] + " " + m[1]); setTimeout(refreshShot, 450); navigator.vibrate && navigator.vibrate(15); }
+        if (m) { send("tapr " + m[0] + " " + m[1]); navigator.vibrate && navigator.vibrate(15); }
       } else {
         dragging = true; send("b left down"); navigator.vibrate && navigator.vibrate(15);
       }
@@ -222,13 +213,11 @@ pad.addEventListener("touchend", e => {
       if (maxFingers < 2) {
         const t = e.changedTouches[0];
         const m = mapToWindow(t.clientX, t.clientY);
-        if (m) { send("tap " + m[0] + " " + m[1]); setTimeout(refreshShot, 450); }
+        if (m) send("tap " + m[0] + " " + m[1]);
       }
     } else {
       send(maxFingers >= 2 ? "b right click" : "b left click");
     }
-  } else if (viewMode && maxFingers >= 2) {
-    setTimeout(refreshShot, 450); // scrolled the window — show the result
   }
   maxFingers = 0;
 }, { passive: false });
@@ -244,19 +233,31 @@ vbtn.addEventListener("click", () => {
   vbtn.classList.toggle("on", viewMode);
   pad.classList.toggle("view", viewMode);
   swHide();
-  if (viewMode) { refreshShot(); shotTimer = setInterval(refreshShot, 1500); }
-  else { clearInterval(shotTimer); }
+  if (viewMode) {
+    const pin = localStorage.getItem("cce-remote-pin") || "";
+    screenEl.src = "/stream?pin=" + encodeURIComponent(pin);
+    send("wl"); // prime winRect
+    rectTimer = setInterval(() => send("wl"), 2000);
+  } else {
+    clearInterval(rectTimer);
+    screenEl.src = "data:,"; // closes the stream connection
+  }
 });
 
 // ── Window switcher ──────────────────────────────────────────────────
 const switcher = document.getElementById("switcher");
+let wantSwitcher = false;
 document.getElementById("wbtn").addEventListener("click", () => {
   if (switcher.classList.contains("show")) { switcher.classList.remove("show"); return; }
+  wantSwitcher = true;
   send("wl"); // reply renders + shows the panel
 });
 function renderSwitcher(json) {
   let wins = [];
   try { wins = JSON.parse(json); } catch (_) {}
+  updateRect(wins); // every wl reply refreshes the tap-mapping rect
+  if (!wantSwitcher) return;
+  wantSwitcher = false;
   // real app windows only: no status-bar modules, no child surfaces
   wins = wins.filter(w => w.mode !== "Status" && !w.has_parent && !w.minimized);
   switcher.innerHTML = "";
diff --git a/src/main.rs b/src/main.rs
index c4c821e..37905ac 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -252,7 +252,76 @@ fn handle_ws(stream: TcpStream, pin: &str) {
     println!("[cce-remote] client disconnected: {peer}");
 }
 
+/// MJPEG stream of the focused window: multipart/x-mixed-replace with one
+/// JPEG part per grim capture (region = the focused window's layout rect,
+/// re-resolved every few frames so the stream follows focus). ~3 fps for a
+/// full-size window — the screencopy dominates, not the encode. Runs until
+/// the client closes the socket. PIN via X-Pin header or ?pin= query (an
+/// <img src> can't carry headers).
+fn handle_stream(mut stream: TcpStream, request_head: &str, pin: &str) {
+    let pin_ok = request_head.lines().any(|l| {
+        let lower = l.to_ascii_lowercase();
+        lower.starts_with("x-pin:") && l[6..].trim() == pin
+    }) || request_head
+        .split_whitespace()
+        .nth(1)
+        .is_some_and(|target| target.contains(&format!("pin={pin}")));
+    if !pin_ok {
+        let _ = write!(stream, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
+        return;
+    }
+    if write!(
+        stream,
+        "HTTP/1.1 200 OK\r\nContent-Type: multipart/x-mixed-replace; boundary=frame\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n"
+    )
+    .is_err()
+    {
+        return;
+    }
+    let mut win = focused_window();
+    let mut tick = 0u32;
+    loop {
+        if tick % 4 == 0 {
+            if let Some(w) = focused_window() {
+                win = Some(w);
+            }
+        }
+        tick = tick.wrapping_add(1);
+        let Some((_, x, y, w, h)) = win else {
+            std::thread::sleep(Duration::from_millis(400));
+            continue;
+        };
+        let out = std::process::Command::new("grim")
+            .args([
+                "-g",
+                &format!("{},{} {}x{}", x as i32, y as i32, w as i32, h as i32),
+                "-t", "jpeg", "-q", "65", "-s", "0.5", "-",
+            ])
+            .output();
+        match out {
+            Ok(o) if o.status.success() && o.stdout.starts_with(&[0xff, 0xd8]) => {
+                let part = format!(
+                    "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: {}\r\n\r\n",
+                    o.stdout.len()
+                );
+                if stream.write_all(part.as_bytes()).is_err()
+                    || stream.write_all(&o.stdout).is_err()
+                    || stream.write_all(b"\r\n").is_err()
+                {
+                    return; // client gone — the loop (and grim spawning) stops
+                }
+            }
+            _ => std::thread::sleep(Duration::from_millis(400)),
+        }
+        std::thread::sleep(Duration::from_millis(40));
+    }
+}
+
 fn handle_http(mut stream: TcpStream, request_head: &str, pin: &str) {
+    if request_head.starts_with("GET /stream") {
+        handle_stream(stream, request_head, pin);
+        return;
+    }
     // /shot: the focused window's screenshot, PIN-gated via the X-Pin header
     // (the page fetch()es it — an <img src> couldn't carry a header).
     if request_head.starts_with("GET /shot") {