734
Pl
pi-linux-use / files
pi-linux-use / .pi / extensions / linux-use.ts
8.0 KB
1import { spawn } from "node:child_process"; 2import { mkdir, readFile } from "node:fs/promises"; 3import { tmpdir } from "node:os"; 4import { join } from "node:path"; 5import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; 6import { StringEnum } from "@earendil-works/pi-ai"; 7import { Type } from "typebox"; 8 9type RunResult = { code: number | null; stdout: string; stderr: string; timedOut: boolean }; 10type LastScreenshot = { path: string; width: number; height: number; timestamp: number }; 11 12let lastScreenshot: LastScreenshot | undefined; 13 14function run(command: string, args: string[], options: { stdin?: string; signal?: AbortSignal; timeoutMs?: number } = {}) { 15 return new Promise<RunResult>((resolve, reject) => { 16 const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] }); 17 const stdout: Buffer[] = []; 18 const stderr: Buffer[] = []; 19 let timedOut = false; 20 21 const kill = () => child.kill("SIGTERM"); 22 options.signal?.addEventListener("abort", kill, { once: true }); 23 24 const timer = options.timeoutMs 25 ? setTimeout(() => { 26 timedOut = true; 27 child.kill("SIGTERM"); 28 setTimeout(() => child.kill("SIGKILL"), 1000).unref(); 29 }, options.timeoutMs) 30 : undefined; 31 32 child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); 33 child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); 34 child.on("error", reject); 35 child.on("close", (code) => { 36 if (timer) clearTimeout(timer); 37 options.signal?.removeEventListener("abort", kill); 38 resolve({ 39 code, 40 stdout: Buffer.concat(stdout).toString("utf8"), 41 stderr: Buffer.concat(stderr).toString("utf8"), 42 timedOut, 43 }); 44 }); 45 46 child.stdin.end(options.stdin ?? ""); 47 }); 48} 49 50function pngSize(buffer: Buffer) { 51 if (buffer.length < 24 || buffer.toString("ascii", 1, 4) !== "PNG") { 52 throw new Error("Expected spectacle to produce a PNG screenshot"); 53 } 54 return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) }; 55} 56 57function requireLastScreenshot() { 58 if (!lastScreenshot) { 59 throw new Error("Call desktop_observe first so x/y pixel coordinates can be mapped to dotool percentages."); 60 } 61 return lastScreenshot; 62} 63 64function pct(value: number, max: number) { 65 const clamped = Math.max(0, Math.min(value, max - 1)); 66 return (clamped / Math.max(1, max - 1)).toFixed(6); 67} 68 69function textToDotoolLines(text: string) { 70 const lines: string[] = []; 71 const parts = text.split("\n"); 72 parts.forEach((part, index) => { 73 if (part.length > 0) lines.push(`type ${part}`); 74 if (index < parts.length - 1) lines.push("key enter"); 75 }); 76 return lines; 77} 78 79export default function (pi: ExtensionAPI) { 80 pi.registerTool({ 81 name: "desktop_observe", 82 label: "Observe Desktop", 83 description: "Capture a full desktop screenshot using KDE Spectacle and return it as an image. Coordinates for desktop_input are pixels in this screenshot.", 84 promptSnippet: "Capture the Linux desktop as an image for GUI computer-use tasks", 85 promptGuidelines: [ 86 "Use desktop_observe before desktop_input when you need to inspect the GUI or choose x/y coordinates.", 87 "desktop_input x/y coordinates are pixels in the latest desktop_observe screenshot.", 88 ], 89 parameters: Type.Object({ 90 includePointer: Type.Optional(Type.Boolean({ description: "Include the mouse pointer in the screenshot. Default false." })), 91 }), 92 async execute(_toolCallId, params, signal, _onUpdate, ctx) { 93 const dir = join(tmpdir(), "pi-linux-use"); 94 await mkdir(dir, { recursive: true }); 95 96 const path = join(dir, `desktop-${Date.now()}.png`); 97 const args = ["-b", "-n", "-f", "-o", path]; 98 if (params.includePointer) args.splice(3, 0, "-p"); 99 100 const result = await run("spectacle", args, { signal, timeoutMs: 15000 }); 101 if (result.code !== 0) { 102 throw new Error(`spectacle failed (${result.code}): ${result.stderr || result.stdout}`); 103 } 104 105 const image = await readFile(path); 106 const size = pngSize(image); 107 lastScreenshot = { path, ...size, timestamp: Date.now() }; 108 109 const supportsImages = ((ctx.model as any)?.input ?? []).includes("image"); 110 let text = `Captured desktop screenshot: ${size.width}x${size.height}px\n`; 111 text += `Path: ${path}\n`; 112 text += "Use desktop_input with x/y pixel coordinates from this image."; 113 if (!supportsImages) { 114 text += "\n\nWarning: the current model does not advertise image input, so only the path/metadata is returned."; 115 } 116 117 return { 118 content: supportsImages 119 ? [ 120 { type: "text" as const, text }, 121 { type: "image" as const, data: image.toString("base64"), mimeType: "image/png" }, 122 ] 123 : [{ type: "text" as const, text }], 124 details: { backend: "spectacle", ...lastScreenshot }, 125 }; 126 }, 127 }); 128 129 pi.registerTool({ 130 name: "desktop_input", 131 label: "Desktop Input", 132 description: "Send basic keyboard and mouse input using dotool. For click/move with x/y, call desktop_observe first and use screenshot pixel coordinates.", 133 promptSnippet: "Send keyboard/mouse input to the Linux desktop via dotool", 134 promptGuidelines: [ 135 "Use desktop_input only after desktop_observe when clicking a visible target; pass x/y in screenshot pixels.", 136 "For dotool key chords use forms like enter, tab, esc, ctrl+l, ctrl+shift+t, alt+f4, super.", 137 ], 138 parameters: Type.Object({ 139 action: StringEnum(["click", "move", "type", "key", "scroll", "script"] as const, { 140 description: "Input action to send.", 141 }), 142 x: Type.Optional(Type.Number({ description: "X pixel coordinate in the latest desktop_observe screenshot." })), 143 y: Type.Optional(Type.Number({ description: "Y pixel coordinate in the latest desktop_observe screenshot." })), 144 button: Type.Optional(StringEnum(["left", "middle", "right"] as const, { description: "Mouse button for click. Default left." })), 145 text: Type.Optional(Type.String({ description: "Text to type for action=type." })), 146 key: Type.Optional(Type.String({ description: "Key/chord for action=key, e.g. enter, ctrl+l, alt+f4." })), 147 amount: Type.Optional(Type.Number({ description: "Wheel amount for action=scroll. Positive usually scrolls up; negative scrolls down." })), 148 script: Type.Optional(Type.String({ description: "Raw newline-separated dotool actions for action=script." })), 149 }), 150 async execute(_toolCallId, params, signal) { 151 const lines: string[] = []; 152 153 const addMoveTo = (x: number, y: number) => { 154 const shot = requireLastScreenshot(); 155 lines.push(`mouseto ${pct(x, shot.width)} ${pct(y, shot.height)}`); 156 }; 157 158 if (params.action === "click") { 159 if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y); 160 lines.push(`click ${params.button ?? "left"}`); 161 } else if (params.action === "move") { 162 if (typeof params.x !== "number" || typeof params.y !== "number") throw new Error("move requires x and y"); 163 addMoveTo(params.x, params.y); 164 } else if (params.action === "type") { 165 if (params.text === undefined) throw new Error("type requires text"); 166 lines.push(...textToDotoolLines(params.text)); 167 } else if (params.action === "key") { 168 if (!params.key) throw new Error("key requires key"); 169 lines.push(`key ${params.key}`); 170 } else if (params.action === "scroll") { 171 if (typeof params.x === "number" && typeof params.y === "number") addMoveTo(params.x, params.y); 172 lines.push(`wheel ${Math.trunc(params.amount ?? -3)}`); 173 } else if (params.action === "script") { 174 if (!params.script) throw new Error("script requires script"); 175 lines.push(params.script); 176 } 177 178 const script = `${lines.join("\n")}\n`; 179 const result = await run("dotool", [], { stdin: script, signal, timeoutMs: 10000 }); 180 if (result.code !== 0) { 181 throw new Error(`dotool failed (${result.code}): ${result.stderr || result.stdout || "Is dotool installed and allowed to write /dev/uinput?"}`); 182 } 183 184 let text = `Sent dotool actions:\n${script.trimEnd()}`; 185 if (result.stderr.trim()) text += `\n\ndotool stderr:\n${result.stderr.trim()}`; 186 return { 187 content: [{ type: "text" as const, text }], 188 details: { backend: "dotool", script, stderr: result.stderr, lastScreenshot }, 189 }; 190 }, 191 }); 192}