xonecas.io
734
Pl

pi-linux-use / commit 8e7de55

feat: working poc

xonecas2026-05-13 8e7de552c7070feaef5ae55e3e09f12a5bea6ce1 parent 5727c43

4 files changed

.pi/extensions/linux-use.ts+192−0
@@ -0,0 +1,192 @@
1+import { spawn } from "node:child_process";
2+import { mkdir, readFile } from "node:fs/promises";
3+import { tmpdir } from "node:os";
4+import { join } from "node:path";
5+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
6+import { StringEnum } from "@earendil-works/pi-ai";
7+import { Type } from "typebox";
8+
9+type RunResult = { code: number | null; stdout: string; stderr: string; timedOut: boolean };
10+type LastScreenshot = { path: string; width: number; height: number; timestamp: number };
11+
12+let lastScreenshot: LastScreenshot | undefined;
13+
14+function run(command: string, args: string[], options: { stdin?: string; signal?: AbortSignal; timeoutMs?: number } = {}) {
15+ return new Promise<RunResult>((resolve, reject) => {
16+ const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
17+ const stdout: Buffer[] = [];
18+ const stderr: Buffer[] = [];
19+ let timedOut = false;
20+
21+ const kill = () => child.kill("SIGTERM");
22+ options.signal?.addEventListener("abort", kill, { once: true });
23+
24+ const timer = options.timeoutMs
25+ ? setTimeout(() => {
26+ timedOut = true;
27+ child.kill("SIGTERM");
28+ setTimeout(() => child.kill("SIGKILL"), 1000).unref();
29+ }, options.timeoutMs)
30+ : undefined;
31+
32+ child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk)));
33+ child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk)));
34+ child.on("error", reject);
35+ child.on("close", (code) => {
36+ if (timer) clearTimeout(timer);
37+ options.signal?.removeEventListener("abort", kill);
38+ resolve({
39+ code,
40+ stdout: Buffer.concat(stdout).toString("utf8"),
41+ stderr: Buffer.concat(stderr).toString("utf8"),
42+ timedOut,
43+ });
44+ });
45+
46+ child.stdin.end(options.stdin ?? "");
47+ });
48+}
49+
50+function pngSize(buffer: Buffer) {
51+ if (buffer.length < 24 || buffer.toString("ascii", 1, 4) !== "PNG") {
52+ throw new Error("Expected spectacle to produce a PNG screenshot");
53+ }
54+ return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
55+}
56+
57+function requireLastScreenshot() {
58+ if (!lastScreenshot) {
59+ throw new Error("Call desktop_observe first so x/y pixel coordinates can be mapped to dotool percentages.");
60+ }
61+ return lastScreenshot;
62+}
63+
64+function pct(value: number, max: number) {
65+ const clamped = Math.max(0, Math.min(value, max - 1));
66+ return (clamped / Math.max(1, max - 1)).toFixed(6);
67+}
68+
69+function textToDotoolLines(text: string) {
70+ const lines: string[] = [];
71+ const parts = text.split("\n");
72+ parts.forEach((part, index) => {
73+ if (part.length > 0) lines.push(`type ${part}`);
74+ if (index < parts.length - 1) lines.push("key enter");
75+ });
76+ return lines;
77+}
78+
79+export default function (pi: ExtensionAPI) {
80+ pi.registerTool({
81+ name: "desktop_observe",
82+ label: "Observe Desktop",
83+ description: "Capture a full desktop screenshot using KDE Spectacle and return it as an image. Coordinates for desktop_input are pixels in this screenshot.",
84+ promptSnippet: "Capture the Linux desktop as an image for GUI computer-use tasks",
85+ promptGuidelines: [
86+ "Use desktop_observe before desktop_input when you need to inspect the GUI or choose x/y coordinates.",
87+ "desktop_input x/y coordinates are pixels in the latest desktop_observe screenshot.",
88+ ],
89+ parameters: Type.Object({
90+ includePointer: Type.Optional(Type.Boolean({ description: "Include the mouse pointer in the screenshot. Default false." })),
91+ }),
92+ async execute(_toolCallId, params, signal, _onUpdate, ctx) {
93+ const dir = join(tmpdir(), "pi-linux-use");
94+ await mkdir(dir, { recursive: true });
95+
96+ const path = join(dir, `desktop-${Date.now()}.png`);
97+ const args = ["-b", "-n", "-f", "-o", path];
98+ if (params.includePointer) args.splice(3, 0, "-p");
99+
100+ const result = await run("spectacle", args, { signal, timeoutMs: 15000 });
101+ if (result.code !== 0) {
102+ throw new Error(`spectacle failed (${result.code}): ${result.stderr || result.stdout}`);
103+ }
104+
105+ const image = await readFile(path);
106+ const size = pngSize(image);
107+ lastScreenshot = { path, ...size, timestamp: Date.now() };
108+
109+ const supportsImages = ((ctx.model as any)?.input ?? []).includes("image");
110+ let text = `Captured desktop screenshot: ${size.width}x${size.height}px\n`;
111+ text += `Path: ${path}\n`;
112+ text += "Use desktop_input with x/y pixel coordinates from this image.";
113+ if (!supportsImages) {
114+ text += "\n\nWarning: the current model does not advertise image input, so only the path/metadata is returned.";
115+ }
116+
117+ return {
118+ content: supportsImages
119+ ? [
120+ { type: "text" as const, text },
121+ { type: "image" as const, data: image.toString("base64"), mimeType: "image/png" },
122+ ]
123+ : [{ type: "text" as const, text }],
124+ details: { backend: "spectacle", ...lastScreenshot },
125+ };
126+ },
127+ });
128+
129+ pi.registerTool({
130+ name: "desktop_input",
131+ label: "Desktop Input",
132+ description: "Send basic keyboard and mouse input using dotool. For click/move with x/y, call desktop_observe first and use screenshot pixel coordinates.",
133+ promptSnippet: "Send keyboard/mouse input to the Linux desktop via dotool",
134+ promptGuidelines: [
135+ "Use desktop_input only after desktop_observe when clicking a visible target; pass x/y in screenshot pixels.",
136+ "For dotool key chords use forms like enter, tab, esc, ctrl+l, ctrl+shift+t, alt+f4, super.",
137+ ],
138+ parameters: Type.Object({
139+ action: StringEnum(["click", "move", "type", "key", "scroll", "script"] as const, {
140+ description: "Input action to send.",
141+ }),
142+ x: Type.Optional(Type.Number({ description: "X pixel coordinate in the latest desktop_observe screenshot." })),
143+ y: Type.Optional(Type.Number({ description: "Y pixel coordinate in the latest desktop_observe screenshot." })),
144+ button: Type.Optional(StringEnum(["left", "middle", "right"] as const, { description: "Mouse button for click. Default left." })),
145+ text: Type.Optional(Type.String({ description: "Text to type for action=type." })),
146+ key: Type.Optional(Type.String({ description: "Key/chord for action=key, e.g. enter, ctrl+l, alt+f4." })),
147+ amount: Type.Optional(Type.Number({ description: "Wheel amount for action=scroll. Positive usually scrolls up; negative scrolls down." })),
148+ script: Type.Optional(Type.String({ description: "Raw newline-separated dotool actions for action=script." })),
149+ }),
150+ async execute(_toolCallId, params, signal) {
151+ const lines: string[] = [];
152+
153+ const addMoveTo = (x: number, y: number) => {
154+ const shot = requireLastScreenshot();
155+ lines.push(`mouseto ${pct(x, shot.width)} ${pct(y, shot.height)}`);
156+ };
157+
158+ if (params.action === "click") {
159+ if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y);
160+ lines.push(`click ${params.button ?? "left"}`);
161+ } else if (params.action === "move") {
162+ if (typeof params.x !== "number" || typeof params.y !== "number") throw new Error("move requires x and y");
163+ addMoveTo(params.x, params.y);
164+ } else if (params.action === "type") {
165+ if (params.text === undefined) throw new Error("type requires text");
166+ lines.push(...textToDotoolLines(params.text));
167+ } else if (params.action === "key") {
168+ if (!params.key) throw new Error("key requires key");
169+ lines.push(`key ${params.key}`);
170+ } else if (params.action === "scroll") {
171+ if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y);
172+ lines.push(`wheel ${Math.trunc(params.amount ?? -3)}`);
173+ } else if (params.action === "script") {
174+ if (!params.script) throw new Error("script requires script");
175+ lines.push(params.script);
176+ }
177+
178+ const script = `${lines.join("\n")}\n`;
179+ const result = await run("dotool", [], { stdin: script, signal, timeoutMs: 10000 });
180+ if (result.code !== 0) {
181+ throw new Error(`dotool failed (${result.code}): ${result.stderr || result.stdout || "Is dotool installed and allowed to write /dev/uinput?"}`);
182+ }
183+
184+ let text = `Sent dotool actions:\n${script.trimEnd()}`;
185+ if (result.stderr.trim()) text += `\n\ndotool stderr:\n${result.stderr.trim()}`;
186+ return {
187+ content: [{ type: "text" as const, text }],
188+ details: { backend: "dotool", script, stderr: result.stderr, lastScreenshot },
189+ };
190+ },
191+ });
192+}
.pi/skills/linux-desktop-use/SKILL.md+72−0
@@ -0,0 +1,72 @@
1+---
2+name: linux-desktop-use
3+description: Use with the pi-linux-use extension to inspect and operate a visible Linux desktop using desktop_observe screenshots and desktop_input dotool actions. Load when the user asks to click, type, scroll, browse a visible website/repo, test GUI behavior, or otherwise interact with the desktop environment.
4+compatibility: Requires the pi-linux-use extension tools desktop_observe and desktop_input, plus KDE Spectacle for screenshots and dotool for input.
5+---
6+
7+# Linux Desktop Use
8+
9+Use this skill when controlling the user's visible Linux desktop through the `desktop_observe` and `desktop_input` tools.
10+
11+## Core loop
12+
13+1. Call `desktop_observe` before any coordinate-based action.
14+2. Inspect the screenshot and choose pixel coordinates in the **full screenshot coordinate space**, not window-relative coordinates.
15+3. Call `desktop_input` for one small action: click, key, type, scroll, move, or script.
16+4. Call `desktop_observe` again to verify the result before continuing.
17+
18+Prefer short observe/action/observe loops over large batches. The screen can move, scroll, or refocus after every action, making old coordinates stale.
19+
20+## Coordinate rules
21+
22+- `desktop_input` x/y coordinates are pixels from the latest `desktop_observe` screenshot.
23+- Coordinates are global desktop coordinates across all monitors.
24+- Re-observe after scrolling, navigation, window focus changes, modal dialogs, or any action that changes layout.
25+- If a target is partly covered by another window, do not guess. Focus/move to a visible target or ask the user.
26+
27+## Safe interaction practices
28+
29+- Click inside the target window before typing, pressing keys, or scrolling.
30+- For scrolling, place the cursor over the scrollable region first. Negative wheel amounts scroll down; positive amounts scroll up.
31+- Use small reversible actions first. Avoid destructive clicks unless the user explicitly asked for them and the target is clearly visible.
32+- Be careful not to type into Pi's own prompt/editor by accident. If focus is uncertain, observe and click the intended application first.
33+- If an action appears to do nothing, assume one of: stale screenshot, wrong focus, cursor over wrong pane, page not loaded, or target obscured. Re-observe and correct.
34+
35+## Browser/repo browsing pattern
36+
37+When the user asks to browse a repo or website already open in the desktop browser, use the visible browser window unless they explicitly ask for separate browser automation.
38+
39+Useful actions:
40+
41+```json
42+{ "action": "click", "x": 3335, "y": 548, "button": "left" }
43+{ "action": "scroll", "x": 4550, "y": 940, "amount": -5 }
44+{ "action": "key", "key": "alt+left" }
45+{ "action": "key", "key": "ctrl+l" }
46+{ "action": "type", "text": "https://example.com" }
47+{ "action": "key", "key": "enter" }
48+```
49+
50+For a visible repository page, first observe the high-level state: URL, repo name, active tab/section, visible files, branches, commits, and any selected file content. Then navigate with clicks or keyboard shortcuts and re-observe.
51+
52+## Key names
53+
54+Use dotool-style key/chord names in `desktop_input`:
55+
56+- `enter`, `tab`, `esc`, `backspace`, `delete`
57+- `ctrl+l`, `ctrl+r`, `ctrl+f`, `ctrl+shift+t`
58+- `alt+left`, `alt+right`, `alt+f4`
59+- `super`
60+
61+## Raw dotool scripts
62+
63+Use `action: "script"` only for simple, deliberate batches where intermediate visual feedback is not needed. Example:
64+
65+```json
66+{
67+ "action": "script",
68+ "script": "key ctrl+l\ntype https://codeberg.org/xonecas/pi-linux-use\nkey enter"
69+}
70+```
71+
72+Do not batch risky clicks or text entry into unknown fields. Observe first, then act.
README.md+87−0
@@ -0,0 +1,87 @@
1+# Pi Linux Use
2+
3+Minimal proof-of-concept Pi extension for Linux desktop computer-use.
4+
5+It gives Pi two tools:
6+
7+- `desktop_observe` — captures the visible desktop with KDE Spectacle and returns a screenshot to the model.
8+- `desktop_input` — sends mouse/keyboard input through `dotool`.
9+
10+A companion skill, `linux-desktop-use`, teaches the agent the observe → act → observe workflow.
11+
12+## Requirements
13+
14+Current POC targets KDE/Plasma Wayland first.
15+
16+Install:
17+
18+- `spectacle` for screenshots
19+- `dotool` for input
20+
21+Verify dotool works:
22+
23+```bash
24+echo type hello | dotool
25+```
26+
27+If dotool cannot access `/dev/uinput`, follow dotool's setup instructions, usually adding your user to the `input` group and reloading udev/rebooting.
28+
29+## Quick start
30+
31+From this repo:
32+
33+```bash
34+pi
35+```
36+
37+Then reload project extensions/skills if Pi is already running:
38+
39+```text
40+/reload
41+```
42+
43+Ask something like:
44+
45+```text
46+Look at my desktop and tell me what is open.
47+```
48+
49+or:
50+
51+```text
52+Browse the repo I have open in the browser.
53+```
54+
55+The agent should use:
56+
57+1. `desktop_observe` to capture the screen.
58+2. `desktop_input` to click, type, scroll, or press keys.
59+3. `desktop_observe` again to verify what changed.
60+
61+## Tool notes
62+
63+Coordinates for `desktop_input` are pixels in the latest `desktop_observe` screenshot. On multi-monitor setups, the screenshot spans the whole desktop.
64+
65+Example input actions:
66+
67+```json
68+{ "action": "click", "x": 3335, "y": 548, "button": "left" }
69+{ "action": "scroll", "x": 4550, "y": 940, "amount": -5 }
70+{ "action": "key", "key": "alt+left" }
71+{ "action": "type", "text": "hello" }
72+```
73+
74+## Using it elsewhere
75+
76+Copy these into another project:
77+
78+```text
79+.pi/extensions/linux-use.ts
80+.pi/skills/linux-desktop-use/SKILL.md
81+```
82+
83+Run `/reload` in Pi.
84+
85+## Status
86+
87+This is intentionally tiny: Spectacle screenshot + dotool input. Future work can add portal ScreenCast/PipeWire, better window metadata, accessibility, OCR, crop/zoom, and wait-for-change helpers.
pi-linux-use.md+136−1
@@ -5,11 +5,146 @@ Idea: build a Pi extension that gives the agent computer-use tools for Linux des
55 Known facts:
66
77 - Pi supports TypeScript extensions that can register custom LLM tools through `pi.registerTool()`.
8+- Pi tool results can include images (`{ type: "image", data, mimeType }`), so an observe tool can return a screenshot directly to vision-capable models.
89 - Linux desktop access depends on the session/compositor. Wayland and X11 require different approaches.
9-- [dotool](https://git.sr.ht/~geb/dotool) gives a full input control api
10+- [dotool](https://git.sr.ht/~geb/dotool) gives a full input control api.
1011
1112 Open questions:
1213
1314 - desktop-agnostic `xdg-desktop-portal` APIs: ScreenCast/PipeWire for observation? How do we let the agent "see"
1415 - does the mvp include a discoverability api? To list open windows, and other data driven operations, this would be an optimization to the simple capture + input base.
1516
17+## Observability layer research
18+
19+Goal: provide the agent with enough visual and structural context to decide where to click/type. Treat observability as layered, not a single API:
20+
21+1. **Raster view**: screenshot/frame image + coordinate system.
22+2. **Display topology**: monitors, scale, global/stream coordinates, cursor position.
23+3. **Window metadata**: active window, titles, bounds, app IDs, z-order where available.
24+4. **Semantic UI**: accessibility tree, OCR, and/or CV-derived elements.
25+5. **Wait/change detection**: wait for screen/window/accessibility changes after actions.
26+
27+### Capture backends
28+
29+#### XDG desktop portals
30+
31+Best cross-desktop Wayland-safe direction, but designed around explicit user consent.
32+
33+- `org.freedesktop.portal.Screenshot`
34+ - `Screenshot(parent_window, options) -> uri`.
35+ - Options include `interactive` and `modal`.
36+ - Good fallback/manual capture path.
37+ - Weakness: likely prompts per capture and has no restore token/session stream, so it is not ideal for frequent agent observe loops.
38+
39+- `org.freedesktop.portal.ScreenCast` + PipeWire
40+ - Flow: `CreateSession -> SelectSources -> Start -> OpenPipeWireRemote`.
41+ - Source types: `MONITOR=1`, `WINDOW=2`, `VIRTUAL=4`.
42+ - Cursor modes: `Hidden=1`, `Embedded=2`, `Metadata=4`.
43+ - `Start` returns PipeWire stream node IDs plus metadata such as `position`, `size`, `source_type`, `id`, `mapping_id`.
44+ - Supports `persist_mode` and one-use `restore_token`, so this is the likely “real” desktop-agnostic observer for a long-running session.
45+ - Implementation note: `OpenPipeWireRemote` returns a Unix FD. A plain `gdbus` shell call is not enough; use a helper/library that can receive the FD and pass it to PipeWire/GStreamer. `gst pipewiresrc` supports `fd` and `path`, so a small Python/Rust helper could expose “latest frame as PNG/JPEG” to the TypeScript extension.
46+
47+- `org.freedesktop.portal.RemoteDesktop`
48+ - Same consent/session model, but includes input devices (`KEYBOARD=1`, `POINTER=2`, `TOUCHSCREEN=4`) and can include ScreenCast sources.
49+ - Potential future replacement for `dotool` on Wayland if we want portal-sanctioned input + observation with matching coordinate spaces.
50+
51+#### Native screenshot helpers
52+
53+Useful for MVP because they are simple and often work without repeated prompts, but they are not universal.
54+
55+- KDE/Plasma Wayland: `spectacle -b -n -f -o <file>` works for full-desktop screenshots; KWin also exposes `org.kde.KWin.ScreenShot2` D-Bus methods.
56+- wlroots compositors: `grim`/`slurp` for screenshots/regions, compositor IPC (`swaymsg`, `hyprctl`) for metadata.
57+- GNOME Wayland: `gnome-screenshot` or GNOME Shell screenshot D-Bus where available; portal ScreenCast is the more portable path.
58+- X11: `xwd`, ImageMagick `import`, `maim`/`scrot` are straightforward; window metadata via EWMH tools.
59+
60+### Discoverability / metadata
61+
62+- **Session detection**: inspect `XDG_SESSION_TYPE`, `WAYLAND_DISPLAY`, `DISPLAY`, `XDG_CURRENT_DESKTOP`, and available commands/services.
63+- **Monitor geometry**:
64+ - X11: `xrandr`.
65+ - KDE: `kscreen-doctor -o`.
66+ - GNOME/wlroots: compositor-specific APIs where available.
67+ - Portal ScreenCast stream metadata gives selected-stream position/size, but note logical compositor coordinates may differ from pixel coordinates under scaling.
68+- **Window list**:
69+ - X11: `wmctrl`, `xdotool`, `xprop`, `xwininfo` via EWMH.
70+ - Wayland: intentionally not globally exposed. Need compositor-specific adapters:
71+ - KDE: KWin scripting/D-Bus can expose windows.
72+ - wlroots: foreign-toplevel protocols via compositor-specific tools.
73+ - GNOME: likely requires Shell extensions/private APIs; do not rely on generic enumeration.
74+ - Therefore window listing should be best-effort and optional, not a prerequisite for MVP.
75+- **Accessibility tree**:
76+ - AT-SPI (`org.a11y.Bus`) can provide roles, names, bounds, actions, focus, and events for apps that support accessibility.
77+ - Cross-toolkit but uneven; may require enabling accessibility. Great semantic layer after screenshot MVP.
78+- **OCR/CV fallback**:
79+ - OCR can turn screenshots into clickable text targets when accessibility/window APIs are unavailable.
80+ - CV can identify buttons, text fields, cursor, and image diffs, but should be additive because raw screenshots are already useful to vision models.
81+
82+### Proposed tool surface
83+
84+MVP tools:
85+
86+- `desktop_observe({ target?: "desktop" | "active_window" | "monitor" | "region", region?, maxWidth?, includeCursor? })`
87+ - Captures one screenshot/frame.
88+ - Returns a text summary plus an image attachment for vision models.
89+ - `details` should include backend, timestamp, image dimensions, monitor layout, coordinate mapping, and temp file path.
90+- `desktop_wait({ condition: "screen_change" | "stable" | "duration", timeoutMs, region? })`
91+ - Lets the agent wait after actions without blind sleeps.
92+
93+Next tools:
94+
95+- `desktop_state({ includeWindows?: boolean, includeA11y?: boolean })`
96+ - Best-effort monitors/windows/focus/accessibility metadata.
97+- `desktop_stream({ action: "start" | "stop" | "status", sourceType?: "monitor" | "window", persist?: boolean })`
98+ - Starts/stops a portal ScreenCast session and stores restore token/session state.
99+- `desktop_zoom({ region, maxWidth? })`
100+ - Crops a region from the latest frame for detailed inspection.
101+
102+### Backend selection sketch
103+
104+1. Detect environment and model image support.
105+2. If Wayland:
106+ - KDE: use `spectacle` for MVP screenshots; later use portal ScreenCast or KWin ScreenShot2 helper.
107+ - wlroots: use `grim`; later portal ScreenCast if available.
108+ - GNOME/unknown: try desktop screenshot helper, else portal Screenshot; for repeated observe, portal ScreenCast helper.
109+3. If X11:
110+ - screenshot via `xwd`/`import`/`maim`;
111+ - metadata via `xrandr` + EWMH tools.
112+4. If text-only model:
113+ - save screenshot path and return OCR/window metadata; optionally ask user/model switch for vision.
114+
115+### Local environment notes from this machine
116+
117+- Current session: `XDG_SESSION_TYPE=wayland`, `XDG_CURRENT_DESKTOP=KDE`, `WAYLAND_DISPLAY=wayland-0`, `DISPLAY=:0`.
118+- Portal services are active: `xdg-desktop-portal`, `plasma-xdg-desktop-portal-kde`, PipeWire, WirePlumber.
119+- Portal exposes:
120+ - `ScreenCast` version 5, `AvailableSourceTypes=7`, `AvailableCursorModes=7`.
121+ - `RemoteDesktop` version 2, `AvailableDeviceTypes=7`.
122+ - `Screenshot` version 2.
123+- `spectacle -b -n -f -o /tmp/pi-observe-test.png` successfully captured a `5120x1440` PNG.
124+- `xprop`/`xdotool` only see an XWayland dummy active window here, confirming that X11 metadata tools are not enough on KDE Wayland.
125+- Python `gi` + GStreamer are available locally, and `gst-inspect-1.0 pipewiresrc` shows `fd` and `path` properties. This makes a portal ScreenCast helper feasible.
126+
127+## Recommended path
128+
129+1. **Build `desktop_observe` first** using pragmatic screenshot helpers and return image tool results. This proves the agent can see and coordinate clicks.
130+2. Add **region crop/zoom and wait-for-change** so the agent can operate iteratively without huge images every turn.
131+3. Add **best-effort metadata** (`desktop_state`) for monitors and windows where available.
132+4. Add a **portal ScreenCast helper** for robust Wayland repeated observation with one user grant.
133+5. Add **AT-SPI/OCR** for semantic targeting once the basic screenshot/action loop works.
134+
135+## Minimal POC
136+
137+A project-local Pi extension lives at `.pi/extensions/linux-use.ts`.
138+
139+A project-local Agent Skill lives at `.pi/skills/linux-desktop-use/SKILL.md` with usage guidance for the screenshot/input loop.
140+
141+Tools:
142+
143+- `desktop_observe` — calls `spectacle -b -n -f -o <tmp.png>`, returns the screenshot as an image attachment when the active model supports image input, and records the image dimensions for coordinate mapping.
144+- `desktop_input` — calls `dotool` with generated stdin actions for click/move/type/key/scroll/raw script. Click/move x/y coordinates are pixels in the latest `desktop_observe` screenshot.
145+
146+Test flow after installing/configuring dotool:
147+
148+1. `/reload` in Pi, or start Pi from this repo so `.pi/extensions/linux-use.ts` is auto-discovered.
149+2. Ask the agent to call `desktop_observe`.
150+3. Ask it to click/type using `desktop_input` with x/y coordinates from the screenshot.