xonecas.io
734
Pl

pi-linux-use / files

pi-linux-use / .pi / extensions / linux-use.ts

8.0 KB
1import { spawn } from "node:child_process";
2import { mkdir, readFile } from "node:fs/promises";
3import { tmpdir } from "node:os";
4import { join } from "node:path";
5import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
6import { StringEnum } from "@earendil-works/pi-ai";
7import { Type } from "typebox";
8
9type RunResult = { code: number | null; stdout: string; stderr: string; timedOut: boolean };
10type LastScreenshot = { path: string; width: number; height: number; timestamp: number };
11
12let lastScreenshot: LastScreenshot | undefined;
13
14function run(command: string, args: string[], options: { stdin?: string; signal?: AbortSignal; timeoutMs?: number } = {}) {
15	return new Promise<RunResult>((resolve, reject) => {
16		const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
17		const stdout: Buffer[] = [];
18		const stderr: Buffer[] = [];
19		let timedOut = false;
20
21		const kill = () => child.kill("SIGTERM");
22		options.signal?.addEventListener("abort", kill, { once: true });
23
24		const timer = options.timeoutMs
25			? setTimeout(() => {
26					timedOut = true;
27					child.kill("SIGTERM");
28					setTimeout(() => child.kill("SIGKILL"), 1000).unref();
29				}, options.timeoutMs)
30			: undefined;
31
32		child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk)));
33		child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk)));
34		child.on("error", reject);
35		child.on("close", (code) => {
36			if (timer) clearTimeout(timer);
37			options.signal?.removeEventListener("abort", kill);
38			resolve({
39				code,
40				stdout: Buffer.concat(stdout).toString("utf8"),
41				stderr: Buffer.concat(stderr).toString("utf8"),
42				timedOut,
43			});
44		});
45
46		child.stdin.end(options.stdin ?? "");
47	});
48}
49
50function pngSize(buffer: Buffer) {
51	if (buffer.length < 24 || buffer.toString("ascii", 1, 4) !== "PNG") {
52		throw new Error("Expected spectacle to produce a PNG screenshot");
53	}
54	return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
55}
56
57function requireLastScreenshot() {
58	if (!lastScreenshot) {
59		throw new Error("Call desktop_observe first so x/y pixel coordinates can be mapped to dotool percentages.");
60	}
61	return lastScreenshot;
62}
63
64function pct(value: number, max: number) {
65	const clamped = Math.max(0, Math.min(value, max - 1));
66	return (clamped / Math.max(1, max - 1)).toFixed(6);
67}
68
69function textToDotoolLines(text: string) {
70	const lines: string[] = [];
71	const parts = text.split("\n");
72	parts.forEach((part, index) => {
73		if (part.length > 0) lines.push(`type ${part}`);
74		if (index < parts.length - 1) lines.push("key enter");
75	});
76	return lines;
77}
78
79export default function (pi: ExtensionAPI) {
80	pi.registerTool({
81		name: "desktop_observe",
82		label: "Observe Desktop",
83		description: "Capture a full desktop screenshot using KDE Spectacle and return it as an image. Coordinates for desktop_input are pixels in this screenshot.",
84		promptSnippet: "Capture the Linux desktop as an image for GUI computer-use tasks",
85		promptGuidelines: [
86			"Use desktop_observe before desktop_input when you need to inspect the GUI or choose x/y coordinates.",
87			"desktop_input x/y coordinates are pixels in the latest desktop_observe screenshot.",
88		],
89		parameters: Type.Object({
90			includePointer: Type.Optional(Type.Boolean({ description: "Include the mouse pointer in the screenshot. Default false." })),
91		}),
92		async execute(_toolCallId, params, signal, _onUpdate, ctx) {
93			const dir = join(tmpdir(), "pi-linux-use");
94			await mkdir(dir, { recursive: true });
95
96			const path = join(dir, `desktop-${Date.now()}.png`);
97			const args = ["-b", "-n", "-f", "-o", path];
98			if (params.includePointer) args.splice(3, 0, "-p");
99
100			const result = await run("spectacle", args, { signal, timeoutMs: 15000 });
101			if (result.code !== 0) {
102				throw new Error(`spectacle failed (${result.code}): ${result.stderr || result.stdout}`);
103			}
104
105			const image = await readFile(path);
106			const size = pngSize(image);
107			lastScreenshot = { path, ...size, timestamp: Date.now() };
108
109			const supportsImages = ((ctx.model as any)?.input ?? []).includes("image");
110			let text = `Captured desktop screenshot: ${size.width}x${size.height}px\n`;
111			text += `Path: ${path}\n`;
112			text += "Use desktop_input with x/y pixel coordinates from this image.";
113			if (!supportsImages) {
114				text += "\n\nWarning: the current model does not advertise image input, so only the path/metadata is returned.";
115			}
116
117			return {
118				content: supportsImages
119					? [
120							{ type: "text" as const, text },
121							{ type: "image" as const, data: image.toString("base64"), mimeType: "image/png" },
122						]
123					: [{ type: "text" as const, text }],
124				details: { backend: "spectacle", ...lastScreenshot },
125			};
126		},
127	});
128
129	pi.registerTool({
130		name: "desktop_input",
131		label: "Desktop Input",
132		description: "Send basic keyboard and mouse input using dotool. For click/move with x/y, call desktop_observe first and use screenshot pixel coordinates.",
133		promptSnippet: "Send keyboard/mouse input to the Linux desktop via dotool",
134		promptGuidelines: [
135			"Use desktop_input only after desktop_observe when clicking a visible target; pass x/y in screenshot pixels.",
136			"For dotool key chords use forms like enter, tab, esc, ctrl+l, ctrl+shift+t, alt+f4, super.",
137		],
138		parameters: Type.Object({
139			action: StringEnum(["click", "move", "type", "key", "scroll", "script"] as const, {
140				description: "Input action to send.",
141			}),
142			x: Type.Optional(Type.Number({ description: "X pixel coordinate in the latest desktop_observe screenshot." })),
143			y: Type.Optional(Type.Number({ description: "Y pixel coordinate in the latest desktop_observe screenshot." })),
144			button: Type.Optional(StringEnum(["left", "middle", "right"] as const, { description: "Mouse button for click. Default left." })),
145			text: Type.Optional(Type.String({ description: "Text to type for action=type." })),
146			key: Type.Optional(Type.String({ description: "Key/chord for action=key, e.g. enter, ctrl+l, alt+f4." })),
147			amount: Type.Optional(Type.Number({ description: "Wheel amount for action=scroll. Positive usually scrolls up; negative scrolls down." })),
148			script: Type.Optional(Type.String({ description: "Raw newline-separated dotool actions for action=script." })),
149		}),
150		async execute(_toolCallId, params, signal) {
151			const lines: string[] = [];
152
153			const addMoveTo = (x: number, y: number) => {
154				const shot = requireLastScreenshot();
155				lines.push(`mouseto ${pct(x, shot.width)} ${pct(y, shot.height)}`);
156			};
157
158			if (params.action === "click") {
159				if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y);
160				lines.push(`click ${params.button ?? "left"}`);
161			} else if (params.action === "move") {
162				if (typeof params.x !== "number" || typeof params.y !== "number") throw new Error("move requires x and y");
163				addMoveTo(params.x, params.y);
164			} else if (params.action === "type") {
165				if (params.text === undefined) throw new Error("type requires text");
166				lines.push(...textToDotoolLines(params.text));
167			} else if (params.action === "key") {
168				if (!params.key) throw new Error("key requires key");
169				lines.push(`key ${params.key}`);
170			} else if (params.action === "scroll") {
171				if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y);
172				lines.push(`wheel ${Math.trunc(params.amount ?? -3)}`);
173			} else if (params.action === "script") {
174				if (!params.script) throw new Error("script requires script");
175				lines.push(params.script);
176			}
177
178			const script = `${lines.join("\n")}\n`;
179			const result = await run("dotool", [], { stdin: script, signal, timeoutMs: 10000 });
180			if (result.code !== 0) {
181				throw new Error(`dotool failed (${result.code}): ${result.stderr || result.stdout || "Is dotool installed and allowed to write /dev/uinput?"}`);
182			}
183
184			let text = `Sent dotool actions:\n${script.trimEnd()}`;
185			if (result.stderr.trim()) text += `\n\ndotool stderr:\n${result.stderr.trim()}`;
186			return {
187				content: [{ type: "text" as const, text }],
188				details: { backend: "dotool", script, stderr: result.stderr, lastScreenshot },
189			};
190		},
191	});
192}