Full remote input parity: forward mouse and structured key events over IPC

- New key_event/mouse_event client messages (crossterm serde) that run through the complete WM event handling on the server — remote clients get window focus, drag/resize and mouse forwarding into virtual terminals (SGR/UTF-8/X10) identical to standalone mode - tui-wm -c now forwards mouse events; previously only keyboard worked remotely - IPC unit tests + end-to-end integration test (tests/integration.py): daemon, frames, click/scroll SGR forwarding, popups, TUI-FM in a window — 12/12 passing - Document the new messages in SOCKET_API.md Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 19:50:27 +02:00
parent 75aa7033a4
commit f0bed7a14e
8 changed files with 373 additions and 63 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -91,6 +91,9 @@ name = "bitflags"
 version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+dependencies = [
+ "serde_core",
+]

 [[package]]
 name = "block-buffer"
@@ -219,6 +222,7 @@ dependencies = [
 "mio",
 "parking_lot",
 "rustix",
+ "serde",
 "signal-hook",
 "signal-hook-mio",
 "winapi",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,7 +8,7 @@ name = "tui-wm"
 path = "src/main.rs"

 [dependencies]
-crossterm = { version = "0.29", features = ["event-stream"] }
+crossterm = { version = "0.29", features = ["event-stream", "serde"] }
 ratatui = "0.30"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
--- a/README.md
+++ b/README.md
@@ -239,7 +239,14 @@ Kör headless — ingen lokal terminal-rendering. State hanteras enbart via sock
 tui-wm -c [socket-sökväg]
 ```

-Ansluter till en befintlig TUI-WM-server (standalone eller daemon) som display-klient. Renderar mottagna frames och vidarebefordrar tangentbords- och resize-events. Om `socket-sökväg` utelämnas används standard-sökvägen. Koppla ned med **Ctrl+Shift+Q**.
+Ansluter till en befintlig TUI-WM-server (standalone eller daemon) som display-klient. Renderar mottagna frames och vidarebefordrar tangentbords-, **mus**- och resize-events — klick, drag, scroll och hover fungerar precis som lokalt, även över SSH. Om `socket-sökväg` utelämnas används standard-sökvägen. Koppla ned med **Ctrl+Shift+Q**.
+
+#### Tester
+
+```bash
+cargo test                       # enhetstester (IPC-protokollet)
+python3 tests/integration.py     # end-to-end: daemon + frames + musforwarding
+```

 ---

--- a/SOCKET_API.md
+++ b/SOCKET_API.md
@@ -85,6 +85,44 @@ Forward raw terminal input bytes to the focused window. Bytes are hex-encoded.

 The example sends the bytes `0x1b 0x5b 0x41` (Up arrow).

+### `key_event`
+
+Forward a structured keyboard event. Unlike `input`, the event runs through
+the full window-manager event handling: global keybinds, dialogs, focus and
+PTY forwarding — exactly as if the key was pressed locally in standalone
+mode. This is what `tui-wm -c` sends for every keypress.
+
+```json
+{
+  "type": "key_event",
+  "event": { "code": { "Char": "a" }, "modifiers": "CONTROL", "kind": "Press", "state": "" }
+}
+```
+
+`code` is either `{ "Char": "<tecken>" }` or a string for special keys
+(`"Enter"`, `"Esc"`, `"Tab"`, `"Backspace"`, `"Up"`, `"Down"`, `"Left"`,
+`"Right"`, `"Home"`, `"End"`, `"PageUp"`, `"PageDown"`, `"Delete"`, ...).
+`modifiers` is a `|`-separated string of `SHIFT`, `CONTROL`, `ALT`
+(empty string = none).
+
+### `mouse_event`
+
+Forward a structured mouse event. Gives remote clients full mouse parity
+with standalone mode: window focus, title-bar drag, resize, and forwarding
+into the virtual terminals with correct SGR/UTF-8/X10 encoding and
+coordinate translation. This is what `tui-wm -c` sends for every mouse event.
+
+```json
+{
+  "type": "mouse_event",
+  "event": { "kind": { "Down": "Left" }, "column": 10, "row": 5, "modifiers": "" }
+}
+```
+
+`kind` is one of `{ "Down": <btn> }`, `{ "Up": <btn> }`, `{ "Drag": <btn> }`,
+`"Moved"`, `"ScrollUp"`, `"ScrollDown"` where `<btn>` is `"Left"`,
+`"Right"` or `"Middle"`. Coordinates are 0-based frame coordinates.
+
 ### `resize`

 Notify the server that the display client's terminal has been resized.
--- a/src/client.rs
+++ b/src/client.rs
@@ -117,12 +117,14 @@ fn client_loop(
                    {
                        break;
                    }
-                    if let Some(bytes) = key_to_bytes(key) {
-                        ipc::write_message(
-                            writer,
-                            &ClientMessage::Input { data: ipc::to_hex(&bytes) },
-                        )?;
-                    }
+                    // Skicka strukturerat så att servern kör eventet genom
+                    // hela WM-hanteringen (globala keybinds, dialoger, PTY).
+                    ipc::write_message(writer, &ClientMessage::KeyEvent { event: key })?;
+                }
+                Event::Mouse(mouse) => {
+                    // Full musforwarding: fokus, drag/resize och vidare-
+                    // befordran in i de virtuella terminalerna.
+                    ipc::write_message(writer, &ClientMessage::MouseEvent { event: mouse })?;
                }
                Event::Resize(w, h) => {
                    term_width = w;
@@ -144,58 +146,3 @@ fn draw_boundary(out: &mut impl Write, at_row: u16, width: u16) -> io::Result<()
    Ok(())
 }

-pub fn key_to_bytes(key: crossterm::event::KeyEvent) -> Option<Vec<u8>> {
-    use crossterm::event::KeyCode::*;
-    let bytes: Vec<u8> = match key.code {
-        Char(c) => {
-            if key.modifiers.contains(KeyModifiers::CONTROL) {
-                match c {
-                    'a'..='z' => vec![c as u8 - b'a' + 1],
-                    'A'..='Z' => vec![c as u8 - b'A' + 1],
-                    '[' => vec![0x1b],
-                    '\\' => vec![0x1c],
-                    ']' => vec![0x1d],
-                    _ => return None,
-                }
-            } else {
-                let mut buf = [0u8; 4];
-                c.encode_utf8(&mut buf).as_bytes().to_vec()
-            }
-        }
-        Enter => vec![b'\r'],
-        Backspace => vec![0x7f],
-        Tab => {
-            if key.modifiers.contains(KeyModifiers::SHIFT) {
-                b"\x1b[Z".to_vec()
-            } else {
-                vec![b'\t']
-            }
-        }
-        Esc => vec![0x1b],
-        Insert => b"\x1b[2~".to_vec(),
-        Up => b"\x1b[A".to_vec(),
-        Down => b"\x1b[B".to_vec(),
-        Right => b"\x1b[C".to_vec(),
-        Left => b"\x1b[D".to_vec(),
-        Home => b"\x1b[H".to_vec(),
-        End => b"\x1b[F".to_vec(),
-        PageUp => b"\x1b[5~".to_vec(),
-        PageDown => b"\x1b[6~".to_vec(),
-        Delete => b"\x1b[3~".to_vec(),
-        F(1) => b"\x1bOP".to_vec(),
-        F(2) => b"\x1bOQ".to_vec(),
-        F(3) => b"\x1bOR".to_vec(),
-        F(4) => b"\x1bOS".to_vec(),
-        F(5) => b"\x1b[15~".to_vec(),
-        F(6) => b"\x1b[17~".to_vec(),
-        F(7) => b"\x1b[18~".to_vec(),
-        F(8) => b"\x1b[19~".to_vec(),
-        F(9) => b"\x1b[20~".to_vec(),
-        F(10) => b"\x1b[21~".to_vec(),
-        F(11) => b"\x1b[23~".to_vec(),
-        F(12) => b"\x1b[24~".to_vec(),
-        BackTab => b"\x1b[Z".to_vec(),
-        _ => return None,
-    };
-    Some(bytes)
-}
--- a/src/ipc.rs
+++ b/src/ipc.rs
@@ -53,6 +53,15 @@ pub enum ClientMessage {
        window_id: Option<usize>,
    },
    Input { data: String },  // hex-encoded raw bytes
+    /// Strukturerat tangentbordsevent från en display-klient.
+    /// Körs genom WM:ns fulla event-hantering (globala keybinds,
+    /// fokus, PTY-forwarding) — till skillnad från Input som skrivs
+    /// rakt in i den fokuserade terminalen.
+    KeyEvent { event: crossterm::event::KeyEvent },
+    /// Strukturerat musevent från en display-klient. Ger fönsterfokus,
+    /// drag/resize och vidarebefordran in i virtuella terminaler
+    /// (SGR/UTF-8/X10-kodning) precis som i standalone-läget.
+    MouseEvent { event: crossterm::event::MouseEvent },
    Resize { width: u16, height: u16 },
    SpawnWindow {
        command: String,
@@ -245,6 +254,78 @@ fn color_fg(c: ratatui::style::Color) -> String {
    }
 }

+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn key_event_roundtrip() {
+        use crossterm::event::{KeyCode, KeyEvent, KeyModifiers};
+        let msg = ClientMessage::KeyEvent {
+            event: KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL),
+        };
+        let json = serde_json::to_string(&msg).unwrap();
+        println!("key_event JSON: {}", json);
+        let back: ClientMessage = serde_json::from_str(&json).unwrap();
+        match back {
+            ClientMessage::KeyEvent { event } => {
+                assert_eq!(event.code, KeyCode::Char('a'));
+                assert_eq!(event.modifiers, KeyModifiers::CONTROL);
+            }
+            _ => panic!("fel variant"),
+        }
+    }
+
+    #[test]
+    fn mouse_event_roundtrip() {
+        use crossterm::event::{MouseButton, MouseEvent, MouseEventKind, KeyModifiers};
+        let msg = ClientMessage::MouseEvent {
+            event: MouseEvent {
+                kind: MouseEventKind::Down(MouseButton::Left),
+                column: 10,
+                row: 5,
+                modifiers: KeyModifiers::NONE,
+            },
+        };
+        let json = serde_json::to_string(&msg).unwrap();
+        println!("mouse_event JSON: {}", json);
+        let back: ClientMessage = serde_json::from_str(&json).unwrap();
+        match back {
+            ClientMessage::MouseEvent { event } => {
+                assert_eq!(event.kind, MouseEventKind::Down(MouseButton::Left));
+                assert_eq!(event.column, 10);
+                assert_eq!(event.row, 5);
+            }
+            _ => panic!("fel variant"),
+        }
+    }
+
+    #[test]
+    fn message_framing_roundtrip() {
+        let msg = ClientMessage::SpawnWindow {
+            command: "htop".to_string(),
+            request_id: Some("t1".to_string()),
+        };
+        let mut buf = Vec::new();
+        write_message(&mut buf, &msg).unwrap();
+        let mut cursor = std::io::Cursor::new(buf);
+        let back: ClientMessage = read_message(&mut cursor).unwrap();
+        match back {
+            ClientMessage::SpawnWindow { command, request_id } => {
+                assert_eq!(command, "htop");
+                assert_eq!(request_id.as_deref(), Some("t1"));
+            }
+            _ => panic!("fel variant"),
+        }
+    }
+
+    #[test]
+    fn hex_roundtrip() {
+        let data = vec![0x00, 0x1b, 0xff, 0x41];
+        assert_eq!(from_hex(&to_hex(&data)), data);
+    }
+}
+
 fn color_bg(c: ratatui::style::Color) -> String {
    use ratatui::style::Color::*;
    match c {
--- a/src/main.rs
+++ b/src/main.rs
@@ -383,6 +383,12 @@ fn handle_ipc_event(
                        }
                    }
                }
+                ClientMessage::KeyEvent { event } => {
+                    app.handle_event(crossterm::event::Event::Key(event));
+                }
+                ClientMessage::MouseEvent { event } => {
+                    app.handle_event(crossterm::event::Event::Mouse(event));
+                }
                ClientMessage::Resize { width, height } => {
                    if let Some(client) = all_clients.get_mut(&client_id) {
                        client.1 = width;
--- a/tests/integration.py
+++ b/tests/integration.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Integrationstest för TUI-WM:s daemon-läge och socket-API.
+
+Startar `tui-wm -d` med en isolerad socket, ansluter som display- och
+app-klient och verifierar:
+
+  1. hello/hello_ok-handskakning och att frames strömmas
+  2. spawn_window + list_windows
+  3. musforwarding: ett klick skickat som `mouse_event` översätts till
+     SGR-mus-protokoll och landar i den virtuella terminalens PTY
+     (verifieras genom att fönstret kör `cat -v` med mus-tracking på,
+     så klicket ekas synligt i frame-innehållet)
+  4. key_event: tangenter går genom WM:ns event-hantering
+  5. spawn_popup + popup_result via Enter
+  6. TUI-FM renderar inne i ett TUI-WM-fönster (om binär finns)
+
+Körning:  python3 tests/integration.py
+Miljö:    TUI_WM_BIN (default target/debug/tui-wm)
+          TUI_FM_BIN (default ../TUI-FM/target/debug/tui-fm, hoppas över om saknas)
+"""
+
+import json
+import os
+import re
+import socket
+import struct
+import subprocess
+import sys
+import tempfile
+import time
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+TUI_WM_BIN = os.environ.get("TUI_WM_BIN", os.path.join(REPO, "target/debug/tui-wm"))
+TUI_FM_BIN = os.environ.get(
+    "TUI_FM_BIN",
+    os.path.normpath(os.path.join(REPO, "../TUI-FM/target/debug/tui-fm")),
+)
+
+ANSI_RE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
+
+passed, failed = [], []
+
+
+def check(name, ok, detail=""):
+    (passed if ok else failed).append(name)
+    print(f"  {'PASS' if ok else 'FAIL'}: {name}" + (f" — {detail}" if detail and not ok else ""))
+
+
+class Conn:
+    def __init__(self, path):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(path)
+        self.sock.settimeout(5.0)
+        self.buf = b""
+
+    def send(self, msg):
+        data = json.dumps(msg).encode()
+        self.sock.sendall(struct.pack(">I", len(data)) + data)
+
+    def recv(self):
+        while len(self.buf) < 4:
+            self.buf += self.sock.recv(65536)
+        (length,) = struct.unpack(">I", self.buf[:4])
+        while len(self.buf) < 4 + length:
+            self.buf += self.sock.recv(65536)
+        payload = self.buf[4 : 4 + length]
+        self.buf = self.buf[4 + length :]
+        return json.loads(payload)
+
+    def recv_until(self, msg_type, timeout=5.0, predicate=None):
+        """Läs meddelanden tills ett av rätt typ (och ev. predikat) dyker upp."""
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            msg = self.recv()
+            if msg.get("type") == msg_type and (predicate is None or predicate(msg)):
+                return msg
+        raise TimeoutError(f"fick aldrig {msg_type}")
+
+    def frame_text(self, timeout=5.0, contains=None):
+        """Läs frames tills texten (ANSI-strippad) innehåller `contains`.
+        Returnerar den strippade texten från senaste framen."""
+        deadline = time.time() + timeout
+        last = ""
+        while time.time() < deadline:
+            msg = self.recv_until("frame", timeout=max(0.1, deadline - time.time()))
+            raw = bytes.fromhex(msg["data"]).decode("utf-8", "replace")
+            last = ANSI_RE.sub("", raw)
+            if contains is None or contains in last:
+                return last
+        return last
+
+
+def key_event(ch=None, code=None, modifiers=""):
+    c = {"Char": ch} if ch else code
+    return {
+        "type": "key_event",
+        "event": {"code": c, "modifiers": modifiers, "kind": "Press", "state": ""},
+    }
+
+
+def mouse_event(kind, col, row):
+    return {
+        "type": "mouse_event",
+        "event": {"kind": kind, "column": col, "row": row, "modifiers": ""},
+    }
+
+
+def main():
+    tmp = tempfile.mkdtemp(prefix="tui-wm-test-")
+    env = dict(os.environ, XDG_RUNTIME_DIR=tmp)
+    sock_path = os.path.join(tmp, "tui-wm.sock")
+
+    daemon = subprocess.Popen(
+        [TUI_WM_BIN, "-d"], cwd=REPO, env=env,
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+    )
+    try:
+        # Vänta på att socketen binds
+        for _ in range(50):
+            if os.path.exists(sock_path):
+                break
+            time.sleep(0.1)
+        check("daemon binder socket", os.path.exists(sock_path))
+
+        # ── 1. Display-klient: handskakning + frames ────────────────────
+        disp = Conn(sock_path)
+        disp.send({"type": "hello", "role": "display", "width": 100, "height": 30})
+        hello = disp.recv()
+        check("hello_ok till display", hello.get("type") == "hello_ok")
+
+        app = Conn(sock_path)
+        app.send({"type": "hello", "role": "app", "width": 0, "height": 0})
+        check("hello_ok till app", app.recv().get("type") == "hello_ok")
+
+        frame = disp.recv_until("frame")
+        check("frames strömmas till display", frame.get("width", 0) > 0)
+
+        # ── 2. spawn_window med mus-tracking-testapp ────────────────────
+        # printf slår på AnyMotion + SGR-mus-tracking, cat -v ekar allt
+        # PTY:n tar emot — inklusive forwardade mus-sekvenser.
+        app.send({
+            "type": "spawn_window",
+            "command": "printf '\\033[?1003h\\033[?1006h'; exec cat -v",
+            "request_id": "w1",
+        })
+        opened = app.recv_until("window_opened")
+        win_id = opened["id"]
+        check("spawn_window svarar window_opened", isinstance(win_id, int))
+
+        app.send({"type": "list_windows", "request_id": "l1"})
+        wl = app.recv_until("window_list")
+        win = next((w for w in wl["windows"] if w["id"] == win_id), None)
+        check("fönstret finns i list_windows", win is not None)
+
+        # ── 3. Musforwarding in i den virtuella terminalen ──────────────
+        time.sleep(0.5)  # låt printf hinna sätta mus-läget
+        cx, cy = win["x"] + 4, win["y"] + 3  # inne i innehållsytan (border=1)
+        disp.send(mouse_event({"Down": "Left"}, cx, cy))
+        disp.send(mouse_event({"Up": "Left"}, cx, cy))
+        text = disp.frame_text(timeout=5.0, contains="[<0;")
+        check("musklick forwardas som SGR till PTY", "[<0;" in text,
+              f"frame-text: {text[-200:]!r}")
+
+        # Scroll ska också forwardas (SGR-knapp 64/65)
+        disp.send(mouse_event("ScrollDown", cx, cy))
+        text = disp.frame_text(timeout=5.0, contains="[<65;")
+        check("scroll forwardas som SGR till PTY", "[<65;" in text,
+              f"frame-text: {text[-200:]!r}")
+
+        # ── 4. key_event genom WM:t (skriv i terminalen) ────────────────
+        disp.send(key_event(ch="x"))
+        text = disp.frame_text(timeout=5.0, contains="x")
+        check("key_event når den virtuella terminalen", "x" in text)
+
+        # ── 5. Popup + popup_result via Enter ───────────────────────────
+        app.send({
+            "type": "spawn_popup", "message": "Integrationstest?",
+            "buttons": ["Ja", "Nej"], "request_id": "p1",
+        })
+        disp.frame_text(timeout=5.0, contains="Integrationstest?")
+        disp.send(key_event(code="Enter"))
+        pr = app.recv_until("popup_result")
+        check("popup_result efter Enter", pr.get("button") == "Ja",
+              f"fick: {pr}")
+
+        # ── 6. TUI-FM inne i TUI-WM ─────────────────────────────────────
+        if os.path.exists(TUI_FM_BIN):
+            app.send({"type": "spawn_window", "command": TUI_FM_BIN,
+                      "request_id": "w2"})
+            fm_open = app.recv_until("window_opened")
+            text = disp.frame_text(timeout=10.0, contains="Namn")
+            check("TUI-FM renderar i TUI-WM-fönster", "Namn" in text,
+                  f"frame-text: {text[-300:]!r}")
+
+            # Klick + scroll in i TUI-FM får inte krascha något
+            app.send({"type": "list_windows", "request_id": "l2"})
+            wl2 = app.recv_until("window_list")
+            fmwin = next((w for w in wl2["windows"] if w["id"] == fm_open["id"]), None)
+            if fmwin:
+                fx, fy = fmwin["x"] + 30, fmwin["y"] + 5
+                disp.send(mouse_event({"Down": "Left"}, fx, fy))
+                disp.send(mouse_event({"Up": "Left"}, fx, fy))
+                disp.send(mouse_event("ScrollDown", fx, fy))
+            time.sleep(0.5)
+            frame = disp.recv_until("frame")
+            check("frames fortsätter efter mus i TUI-FM", frame.get("width", 0) > 0)
+        else:
+            print(f"  SKIP: TUI-FM-binär saknas ({TUI_FM_BIN})")
+
+        # ── Städa: stäng fönster ────────────────────────────────────────
+        app.send({"type": "close_window", "window_id": win_id})
+
+    finally:
+        daemon.terminate()
+        try:
+            daemon.wait(timeout=3)
+        except subprocess.TimeoutExpired:
+            daemon.kill()
+
+    print(f"\n{len(passed)} PASS, {len(failed)} FAIL")
+    if failed:
+        print("Misslyckade: " + ", ".join(failed))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()