remote trackpad and keyboard server
git clone https://git.lucas.co/cce-remote.git
feat: window view is a live MJPEG stream
/stream serves multipart/x-mixed-replace JPEG parts from grim region
captures of the focused window (half scale, q65; the region re-resolves
every few frames so the stream follows focus; loop ends when the client
closes). ~2.5 fps for a full-size window — the screencopy dominates.
The page renders it natively in the <img> (PIN via ?pin= query, since
img can't carry headers); winRect for tap mapping is kept fresh by
polling wl over the WS, and the switcher renders only on request so
those polls stay silent. Post-interaction refresh timers are gone —
the stream IS the feedback.
Co-Authored-By: Claude Fable 5 <[email protected]>
README.md | 10 ++++-----
index.html | 47 ++++++++++++++++++++---------------------
src/main.rs | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 98 insertions(+), 28 deletions(-)
diff --git a/README.md b/README.md
index 0af455a..d191b95 100644
--- a/README.md
+++ b/README.md
@@ -30,11 +30,11 @@ Home Screen for a fullscreen app feel.
char→evdev map; iOS `beforeinput` is used, so autocorrect noise is
filtered)
- ☰ — window switcher: tap a window to focus it
-- 🖥 — window view mode: the pad shows the focused window (auto-refreshing
- screenshot, ~1.5s + after each interaction). Tap to click that spot,
- long-press to right-click, two-finger drag to scroll. Toggle again for
- the trackpad. Screenshots are served PIN-gated and deleted immediately —
- nothing accumulates on disk.
+- 🖥 — window view mode: a live MJPEG stream of the focused window
+ (~2-3 fps, grim region captures at half scale; the stream follows focus).
+ Tap to click that spot, long-press to right-click, two-finger drag to
+ scroll. Toggle again for the trackpad. Both `/stream` and the one-shot
+ `/shot` endpoint are PIN-gated; nothing accumulates on disk.
## Security
diff --git a/index.html b/index.html
index 5b7b4f9..9fce00f 100644
--- a/index.html
+++ b/index.html
@@ -124,24 +124,15 @@ const ACCEL = 1.6, SCROLL = 0.8;
let touches = new Map(), moved = 0, startT = 0, maxFingers = 0, startX = 0, startY = 0;
let dragging = false, dragTimer = null, pendDx = 0, pendDy = 0, pendSy = 0, flushT = null;
-// ── Window view mode: the pad shows the focused window; taps click it ──
-let viewMode = false, winRect = null, shotTimer = null, shotBusy = false;
+// ── Window view mode: live MJPEG stream of the focused window; taps
+// click it. The <img> renders /stream (multipart/x-mixed-replace)
+// natively; winRect for tap mapping is kept fresh by polling `wl` over
+// the WS (the stream follows focus server-side).
+let viewMode = false, winRect = null, rectTimer = null;
const screenEl = document.getElementById("screen");
-async function refreshShot() {
- if (!viewMode || shotBusy) return;
- shotBusy = true;
- try {
- const r = await fetch("/shot", { headers: { "X-Pin": localStorage.getItem("cce-remote-pin") || "" }, cache: "no-store" });
- if (r.ok) {
- const xw = (r.headers.get("X-Win") || "").split(" ").map(Number);
- if (xw.length === 5) winRect = { id: xw[0], x: xw[1], y: xw[2], w: xw[3], h: xw[4] };
- const url = URL.createObjectURL(await r.blob());
- const old = screenEl.src;
- screenEl.onload = () => { if (old && old.startsWith("blob:")) URL.revokeObjectURL(old); };
- screenEl.src = url;
- }
- } catch (_) {}
- shotBusy = false;
+function updateRect(wins) {
+ const f = wins.find(w => w.focused && w.mode !== "Status");
+ if (f) winRect = { id: f.id, x: f.x, y: f.y, w: f.w, h: f.h };
}
// touch point (viewport px) → layout px inside the focused window,
// accounting for the contain-fit letterbox. null when off the image.
@@ -180,7 +171,7 @@ pad.addEventListener("touchstart", e => {
if (viewMode) {
// long-press in view mode = right click at the spot
const m = mapToWindow(startX, startY);
- if (m) { send("tapr " + m[0] + " " + m[1]); setTimeout(refreshShot, 450); navigator.vibrate && navigator.vibrate(15); }
+ if (m) { send("tapr " + m[0] + " " + m[1]); navigator.vibrate && navigator.vibrate(15); }
} else {
dragging = true; send("b left down"); navigator.vibrate && navigator.vibrate(15);
}
@@ -222,13 +213,11 @@ pad.addEventListener("touchend", e => {
if (maxFingers < 2) {
const t = e.changedTouches[0];
const m = mapToWindow(t.clientX, t.clientY);
- if (m) { send("tap " + m[0] + " " + m[1]); setTimeout(refreshShot, 450); }
+ if (m) send("tap " + m[0] + " " + m[1]);
}
} else {
send(maxFingers >= 2 ? "b right click" : "b left click");
}
- } else if (viewMode && maxFingers >= 2) {
- setTimeout(refreshShot, 450); // scrolled the window — show the result
}
maxFingers = 0;
}, { passive: false });
@@ -244,19 +233,31 @@ vbtn.addEventListener("click", () => {
vbtn.classList.toggle("on", viewMode);
pad.classList.toggle("view", viewMode);
swHide();
- if (viewMode) { refreshShot(); shotTimer = setInterval(refreshShot, 1500); }
- else { clearInterval(shotTimer); }
+ if (viewMode) {
+ const pin = localStorage.getItem("cce-remote-pin") || "";
+ screenEl.src = "/stream?pin=" + encodeURIComponent(pin);
+ send("wl"); // prime winRect
+ rectTimer = setInterval(() => send("wl"), 2000);
+ } else {
+ clearInterval(rectTimer);
+ screenEl.src = "data:,"; // closes the stream connection
+ }
});
// ── Window switcher ──────────────────────────────────────────────────
const switcher = document.getElementById("switcher");
+let wantSwitcher = false;
document.getElementById("wbtn").addEventListener("click", () => {
if (switcher.classList.contains("show")) { switcher.classList.remove("show"); return; }
+ wantSwitcher = true;
send("wl"); // reply renders + shows the panel
});
function renderSwitcher(json) {
let wins = [];
try { wins = JSON.parse(json); } catch (_) {}
+ updateRect(wins); // every wl reply refreshes the tap-mapping rect
+ if (!wantSwitcher) return;
+ wantSwitcher = false;
// real app windows only: no status-bar modules, no child surfaces
wins = wins.filter(w => w.mode !== "Status" && !w.has_parent && !w.minimized);
switcher.innerHTML = "";
diff --git a/src/main.rs b/src/main.rs
index c4c821e..37905ac 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -252,7 +252,76 @@ fn handle_ws(stream: TcpStream, pin: &str) {
println!("[cce-remote] client disconnected: {peer}");
}
+/// MJPEG stream of the focused window: multipart/x-mixed-replace with one
+/// JPEG part per grim capture (region = the focused window's layout rect,
+/// re-resolved every few frames so the stream follows focus). ~3 fps for a
+/// full-size window — the screencopy dominates, not the encode. Runs until
+/// the client closes the socket. PIN via X-Pin header or ?pin= query (an
+/// <img src> can't carry headers).
+fn handle_stream(mut stream: TcpStream, request_head: &str, pin: &str) {
+ let pin_ok = request_head.lines().any(|l| {
+ let lower = l.to_ascii_lowercase();
+ lower.starts_with("x-pin:") && l[6..].trim() == pin
+ }) || request_head
+ .split_whitespace()
+ .nth(1)
+ .is_some_and(|target| target.contains(&format!("pin={pin}")));
+ if !pin_ok {
+ let _ = write!(stream, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
+ return;
+ }
+ if write!(
+ stream,
+ "HTTP/1.1 200 OK\r\nContent-Type: multipart/x-mixed-replace; boundary=frame\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n"
+ )
+ .is_err()
+ {
+ return;
+ }
+ let mut win = focused_window();
+ let mut tick = 0u32;
+ loop {
+ if tick % 4 == 0 {
+ if let Some(w) = focused_window() {
+ win = Some(w);
+ }
+ }
+ tick = tick.wrapping_add(1);
+ let Some((_, x, y, w, h)) = win else {
+ std::thread::sleep(Duration::from_millis(400));
+ continue;
+ };
+ let out = std::process::Command::new("grim")
+ .args([
+ "-g",
+ &format!("{},{} {}x{}", x as i32, y as i32, w as i32, h as i32),
+ "-t", "jpeg", "-q", "65", "-s", "0.5", "-",
+ ])
+ .output();
+ match out {
+ Ok(o) if o.status.success() && o.stdout.starts_with(&[0xff, 0xd8]) => {
+ let part = format!(
+ "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: {}\r\n\r\n",
+ o.stdout.len()
+ );
+ if stream.write_all(part.as_bytes()).is_err()
+ || stream.write_all(&o.stdout).is_err()
+ || stream.write_all(b"\r\n").is_err()
+ {
+ return; // client gone — the loop (and grim spawning) stops
+ }
+ }
+ _ => std::thread::sleep(Duration::from_millis(400)),
+ }
+ std::thread::sleep(Duration::from_millis(40));
+ }
+}
+
fn handle_http(mut stream: TcpStream, request_head: &str, pin: &str) {
+ if request_head.starts_with("GET /stream") {
+ handle_stream(stream, request_head, pin);
+ return;
+ }
// /shot: the focused window's screenshot, PIN-gated via the X-Pin header
// (the page fetch()es it — an <img src> couldn't carry a header).
if request_head.starts_with("GET /shot") {