@@ -0,0 +1,192 @@
1+import { spawn } from "node:child_process";
2+import { mkdir, readFile } from "node:fs/promises";
3+import { tmpdir } from "node:os";
4+import { join } from "node:path";
5+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
6+import { StringEnum } from "@earendil-works/pi-ai";
7+import { Type } from "typebox";
8+
9+type RunResult = { code: number | null; stdout: string; stderr: string; timedOut: boolean };
10+type LastScreenshot = { path: string; width: number; height: number; timestamp: number };
11+
12+let lastScreenshot: LastScreenshot | undefined;
13+
14+function run(command: string, args: string[], options: { stdin?: string; signal?: AbortSignal; timeoutMs?: number } = {}) {
15+ return new Promise<RunResult>((resolve, reject) => {
16+ const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
17+ const stdout: Buffer[] = [];
18+ const stderr: Buffer[] = [];
19+ let timedOut = false;
20+
21+ const kill = () => child.kill("SIGTERM");
22+ options.signal?.addEventListener("abort", kill, { once: true });
23+
24+ const timer = options.timeoutMs
25+ ? setTimeout(() => {
26+ timedOut = true;
27+ child.kill("SIGTERM");
28+ setTimeout(() => child.kill("SIGKILL"), 1000).unref();
29+ }, options.timeoutMs)
30+ : undefined;
31+
32+ child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk)));
33+ child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk)));
34+ child.on("error", reject);
35+ child.on("close", (code) => {
36+ if (timer) clearTimeout(timer);
37+ options.signal?.removeEventListener("abort", kill);
38+ resolve({
39+ code,
40+ stdout: Buffer.concat(stdout).toString("utf8"),
41+ stderr: Buffer.concat(stderr).toString("utf8"),
42+ timedOut,
43+ });
44+ });
45+
46+ child.stdin.end(options.stdin ?? "");
47+ });
48+}
49+
50+function pngSize(buffer: Buffer) {
51+ if (buffer.length < 24 || buffer.toString("ascii", 1, 4) !== "PNG") {
52+ throw new Error("Expected spectacle to produce a PNG screenshot");
53+ }
54+ return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
55+}
56+
57+function requireLastScreenshot() {
58+ if (!lastScreenshot) {
59+ throw new Error("Call desktop_observe first so x/y pixel coordinates can be mapped to dotool percentages.");
60+ }
61+ return lastScreenshot;
62+}
63+
64+function pct(value: number, max: number) {
65+ const clamped = Math.max(0, Math.min(value, max - 1));
66+ return (clamped / Math.max(1, max - 1)).toFixed(6);
67+}
68+
69+function textToDotoolLines(text: string) {
70+ const lines: string[] = [];
71+ const parts = text.split("\n");
72+ parts.forEach((part, index) => {
73+ if (part.length > 0) lines.push(`type ${part}`);
74+ if (index < parts.length - 1) lines.push("key enter");
75+ });
76+ return lines;
77+}
78+
79+export default function (pi: ExtensionAPI) {
80+ pi.registerTool({
81+ name: "desktop_observe",
82+ label: "Observe Desktop",
83+ description: "Capture a full desktop screenshot using KDE Spectacle and return it as an image. Coordinates for desktop_input are pixels in this screenshot.",
84+ promptSnippet: "Capture the Linux desktop as an image for GUI computer-use tasks",
85+ promptGuidelines: [
86+ "Use desktop_observe before desktop_input when you need to inspect the GUI or choose x/y coordinates.",
87+ "desktop_input x/y coordinates are pixels in the latest desktop_observe screenshot.",
88+ ],
89+ parameters: Type.Object({
90+ includePointer: Type.Optional(Type.Boolean({ description: "Include the mouse pointer in the screenshot. Default false." })),
91+ }),
92+ async execute(_toolCallId, params, signal, _onUpdate, ctx) {
93+ const dir = join(tmpdir(), "pi-linux-use");
94+ await mkdir(dir, { recursive: true });
95+
96+ const path = join(dir, `desktop-${Date.now()}.png`);
97+ const args = ["-b", "-n", "-f", "-o", path];
98+ if (params.includePointer) args.splice(3, 0, "-p");
99+
100+ const result = await run("spectacle", args, { signal, timeoutMs: 15000 });
101+ if (result.code !== 0) {
102+ throw new Error(`spectacle failed (${result.code}): ${result.stderr || result.stdout}`);
103+ }
104+
105+ const image = await readFile(path);
106+ const size = pngSize(image);
107+ lastScreenshot = { path, ...size, timestamp: Date.now() };
108+
109+ const supportsImages = ((ctx.model as any)?.input ?? []).includes("image");
110+ let text = `Captured desktop screenshot: ${size.width}x${size.height}px\n`;
111+ text += `Path: ${path}\n`;
112+ text += "Use desktop_input with x/y pixel coordinates from this image.";
113+ if (!supportsImages) {
114+ text += "\n\nWarning: the current model does not advertise image input, so only the path/metadata is returned.";
115+ }
116+
117+ return {
118+ content: supportsImages
119+ ? [
120+ { type: "text" as const, text },
121+ { type: "image" as const, data: image.toString("base64"), mimeType: "image/png" },
122+ ]
123+ : [{ type: "text" as const, text }],
124+ details: { backend: "spectacle", ...lastScreenshot },
125+ };
126+ },
127+ });
128+
129+ pi.registerTool({
130+ name: "desktop_input",
131+ label: "Desktop Input",
132+ description: "Send basic keyboard and mouse input using dotool. For click/move with x/y, call desktop_observe first and use screenshot pixel coordinates.",
133+ promptSnippet: "Send keyboard/mouse input to the Linux desktop via dotool",
134+ promptGuidelines: [
135+ "Use desktop_input only after desktop_observe when clicking a visible target; pass x/y in screenshot pixels.",
136+ "For dotool key chords use forms like enter, tab, esc, ctrl+l, ctrl+shift+t, alt+f4, super.",
137+ ],
138+ parameters: Type.Object({
139+ action: StringEnum(["click", "move", "type", "key", "scroll", "script"] as const, {
140+ description: "Input action to send.",
141+ }),
142+ x: Type.Optional(Type.Number({ description: "X pixel coordinate in the latest desktop_observe screenshot." })),
143+ y: Type.Optional(Type.Number({ description: "Y pixel coordinate in the latest desktop_observe screenshot." })),
144+ button: Type.Optional(StringEnum(["left", "middle", "right"] as const, { description: "Mouse button for click. Default left." })),
145+ text: Type.Optional(Type.String({ description: "Text to type for action=type." })),
146+ key: Type.Optional(Type.String({ description: "Key/chord for action=key, e.g. enter, ctrl+l, alt+f4." })),
147+ amount: Type.Optional(Type.Number({ description: "Wheel amount for action=scroll. Positive usually scrolls up; negative scrolls down." })),
148+ script: Type.Optional(Type.String({ description: "Raw newline-separated dotool actions for action=script." })),
149+ }),
150+ async execute(_toolCallId, params, signal) {
151+ const lines: string[] = [];
152+
153+ const addMoveTo = (x: number, y: number) => {
154+ const shot = requireLastScreenshot();
155+ lines.push(`mouseto ${pct(x, shot.width)} ${pct(y, shot.height)}`);
156+ };
157+
158+ if (params.action === "click") {
159+ if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y);
160+ lines.push(`click ${params.button ?? "left"}`);
161+ } else if (params.action === "move") {
162+ if (typeof params.x !== "number" || typeof params.y !== "number") throw new Error("move requires x and y");
163+ addMoveTo(params.x, params.y);
164+ } else if (params.action === "type") {
165+ if (params.text === undefined) throw new Error("type requires text");
166+ lines.push(...textToDotoolLines(params.text));
167+ } else if (params.action === "key") {
168+ if (!params.key) throw new Error("key requires key");
169+ lines.push(`key ${params.key}`);
170+ } else if (params.action === "scroll") {
171+ if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y);
172+ lines.push(`wheel ${Math.trunc(params.amount ?? -3)}`);
173+ } else if (params.action === "script") {
174+ if (!params.script) throw new Error("script requires script");
175+ lines.push(params.script);
176+ }
177+
178+ const script = `${lines.join("\n")}\n`;
179+ const result = await run("dotool", [], { stdin: script, signal, timeoutMs: 10000 });
180+ if (result.code !== 0) {
181+ throw new Error(`dotool failed (${result.code}): ${result.stderr || result.stdout || "Is dotool installed and allowed to write /dev/uinput?"}`);
182+ }
183+
184+ let text = `Sent dotool actions:\n${script.trimEnd()}`;
185+ if (result.stderr.trim()) text += `\n\ndotool stderr:\n${result.stderr.trim()}`;
186+ return {
187+ content: [{ type: "text" as const, text }],
188+ details: { backend: "dotool", script, stderr: result.stderr, lastScreenshot },
189+ };
190+ },
191+ });
192+}