xonecas.io
64875
Mc

mini-coder / commit 69e87ad

feat: handle inline file generation from multimodal models

When a multimodal model (GPT-5, Gemini 3.x Flash Image) produces images
inline during streamText, save them to disk and display a one-liner.

Changes:
- New generated-files.ts: extensionFromMediaType + saveGeneratedFile helpers
- New FileGeneratedEvent turn event type
- mapFullStreamToTurnEvents handles "file" stream chunks, saves via
  saveGeneratedFile, yields file-generated events
- stream-render.ts renders: · file  image/png  →  generated-1.png
- cwd threaded from SessionRunner → runTurn → mapFullStreamToTurnEvents
- Design decision documented in docs/design-decisions.md

xonecas2026-03-22 69e87ad90359fc4260b88f616e4964673ca87acb parent aabd4ea

9 files changed

src/agent/session-runner.ts+1−0
@@ -214,6 +214,7 @@ export class SessionRunner {
214214 ...(this.currentThinkingEffort
215215 ? { thinkingEffort: this.currentThinkingEffort }
216216 : {}),
217+ cwd: this.cwd,
217218 });
218219
219220 const { inputTokens, outputTokens, contextTokens, newMessages } =
src/cli/stream-render.test.ts+20−0
@@ -349,4 +349,24 @@ describe("renderTurn", () => {
349349 const plain = stripAnsi(getCapturedStdout());
350350 expect(countOccurrences(plain, "$ printf ok")).toBe(1);
351351 });
352+
353+ test("renders file-generated events as a one-liner", async () => {
354+ captureStdout();
355+
356+ await renderTurn(
357+ eventsFrom([
358+ {
359+ type: "file-generated",
360+ filePath: "/home/user/project/generated-1.png",
361+ mediaType: "image/png",
362+ },
363+ done(),
364+ ]),
365+ new Spinner(),
366+ );
367+
368+ const plain = stripAnsi(getCapturedStdout());
369+ expect(plain).toContain("image/png");
370+ expect(plain).toContain("generated-1.png");
371+ });
352372 });
src/cli/stream-render.ts+15−0
@@ -1,3 +1,4 @@
1+import { basename } from "node:path";
12 import * as c from "yoctocolors";
23 import { buildAbortMessages, isAbortError } from "../agent/agent-helpers.ts";
34 import type { CoreMessage } from "../llm-api/turn.ts";
@@ -137,6 +138,20 @@ export async function renderTurn(
137138 break;
138139 }
139140
141+ case "file-generated": {
142+ liveReasoning.finish();
143+ content.flushOpenContent();
144+ if (!quiet) {
145+ spinner.stop();
146+ if (renderedVisibleOutput) writeln();
147+ writeln(
148+ `${G.info} ${c.dim("file")} ${c.dim(event.mediaType)} ${c.dim("→")} ${basename(event.filePath)}`,
149+ );
150+ renderedVisibleOutput = true;
151+ }
152+ break;
153+ }
154+
140155 case "turn-complete": {
141156 liveReasoning.finish();
142157 content.flushOpenContent();
src/llm-api/generated-files.test.ts+72−0
@@ -0,0 +1,72 @@
1+import { describe, expect, test } from "bun:test";
2+import { join } from "node:path";
3+import {
4+ extensionFromMediaType,
5+ saveGeneratedFile,
6+} from "./generated-files.ts";
7+
8+describe("extensionFromMediaType", () => {
9+ test("maps common image types", () => {
10+ expect(extensionFromMediaType("image/png")).toBe("png");
11+ expect(extensionFromMediaType("image/jpeg")).toBe("jpg");
12+ expect(extensionFromMediaType("image/gif")).toBe("gif");
13+ expect(extensionFromMediaType("image/webp")).toBe("webp");
14+ expect(extensionFromMediaType("image/svg+xml")).toBe("svg");
15+ });
16+
17+ test("extracts subtype for unknown types", () => {
18+ expect(extensionFromMediaType("image/bmp")).toBe("bmp");
19+ expect(extensionFromMediaType("image/tiff")).toBe("tiff");
20+ expect(extensionFromMediaType("audio/mp3")).toBe("mp3");
21+ });
22+
23+ test("returns bin for unparseable types", () => {
24+ expect(extensionFromMediaType("")).toBe("bin");
25+ expect(extensionFromMediaType("invalid")).toBe("bin");
26+ });
27+});
28+
29+describe("saveGeneratedFile", () => {
30+ test("saves file to disk and returns path", async () => {
31+ const dir = join(import.meta.dir, "../../.test-tmp-generated");
32+ await Bun.write(join(dir, ".gitkeep"), "");
33+
34+ const data = new Uint8Array([137, 80, 78, 71]); // PNG magic bytes
35+ const filePath = await saveGeneratedFile(
36+ { mediaType: "image/png", uint8Array: data },
37+ dir,
38+ );
39+
40+ expect(filePath).toMatch(/generated-\d+\.png$/);
41+ const file = Bun.file(filePath);
42+ expect(await file.exists()).toBe(true);
43+ expect(new Uint8Array(await file.arrayBuffer())).toEqual(data);
44+
45+ // cleanup
46+ const { rm } = await import("node:fs/promises");
47+ await rm(dir, { recursive: true });
48+ });
49+
50+ test("increments counter to avoid collisions", async () => {
51+ const dir = join(import.meta.dir, "../../.test-tmp-generated2");
52+ await Bun.write(join(dir, ".gitkeep"), "");
53+
54+ const data = new Uint8Array([0xff, 0xd8]); // JPEG magic
55+ const path1 = await saveGeneratedFile(
56+ { mediaType: "image/jpeg", uint8Array: data },
57+ dir,
58+ );
59+ const path2 = await saveGeneratedFile(
60+ { mediaType: "image/jpeg", uint8Array: data },
61+ dir,
62+ );
63+
64+ expect(path1).not.toBe(path2);
65+ expect(await Bun.file(path1).exists()).toBe(true);
66+ expect(await Bun.file(path2).exists()).toBe(true);
67+
68+ // cleanup
69+ const { rm } = await import("node:fs/promises");
70+ await rm(dir, { recursive: true });
71+ });
72+});
src/llm-api/generated-files.ts+34−0
@@ -0,0 +1,34 @@
1+import { join } from "node:path";
2+
3+const MEDIA_TYPE_TO_EXT: Record<string, string> = {
4+ "image/jpeg": "jpg",
5+ "image/svg+xml": "svg",
6+};
7+
8+/** Derive a file extension from a MIME media type. */
9+export function extensionFromMediaType(mediaType: string): string {
10+ if (MEDIA_TYPE_TO_EXT[mediaType]) return MEDIA_TYPE_TO_EXT[mediaType];
11+ const slash = mediaType.indexOf("/");
12+ if (slash === -1 || slash === mediaType.length - 1) return "bin";
13+ return mediaType.slice(slash + 1);
14+}
15+
16+interface GeneratedFileData {
17+ mediaType: string;
18+ uint8Array: Uint8Array;
19+}
20+
21+let counter = 0;
22+
23+/** Save a generated file to disk, returning the absolute path. */
24+export async function saveGeneratedFile(
25+ file: GeneratedFileData,
26+ cwd: string,
27+): Promise<string> {
28+ counter += 1;
29+ const ext = extensionFromMediaType(file.mediaType);
30+ const name = `generated-${counter}.${ext}`;
31+ const filePath = join(cwd, name);
32+ await Bun.write(filePath, file.uint8Array);
33+ return filePath;
34+}
src/llm-api/turn-execution.test.ts+36−1
@@ -14,9 +14,12 @@ function chunksFrom(
1414
1515 async function collectEvents(
1616 chunks: Array<{ type?: string; [key: string]: unknown }>,
17+ opts?: { cwd?: string },
1718 ): Promise<TurnEvent[]> {
1819 const events: TurnEvent[] = [];
19- for await (const event of mapFullStreamToTurnEvents(chunksFrom(chunks), {})) {
20+ for await (const event of mapFullStreamToTurnEvents(chunksFrom(chunks), {
21+ ...(opts?.cwd ? { cwd: opts.cwd } : {}),
22+ })) {
2023 events.push(event);
2124 }
2225 return events;
@@ -254,4 +257,36 @@ describe("mapFullStreamToTurnEvents", () => {
254257 { type: "text-delta", delta: " visible" },
255258 ]);
256259 });
260+
261+ test("handles file stream chunks by saving to disk and yielding file-generated event", async () => {
262+ const { join } = await import("node:path");
263+ const { rm } = await import("node:fs/promises");
264+ const dir = join(import.meta.dir, "../../.test-tmp-file-chunks");
265+ await Bun.write(join(dir, ".gitkeep"), "");
266+
267+ const events = await collectEvents(
268+ [
269+ {
270+ type: "file",
271+ file: {
272+ mediaType: "image/png",
273+ uint8Array: new Uint8Array([137, 80, 78, 71]),
274+ base64: "",
275+ },
276+ },
277+ ],
278+ { cwd: dir },
279+ );
280+
281+ expect(events).toHaveLength(1);
282+ expect(events[0]).toMatchObject({
283+ type: "file-generated",
284+ mediaType: "image/png",
285+ });
286+ const evt = events[0] as Extract<TurnEvent, { type: "file-generated" }>;
287+ expect(evt.filePath).toMatch(/generated-\d+\.png$/);
288+ expect(await Bun.file(evt.filePath).exists()).toBe(true);
289+
290+ await rm(dir, { recursive: true });
291+ });
257292 });
src/llm-api/turn-execution.ts+17−0
@@ -7,6 +7,7 @@
77 import type { FlexibleSchema, StepResult } from "ai";
88 import { dynamicTool, jsonSchema, type streamText } from "ai";
99 import { normalizeUnknownError } from "./error-utils.ts";
10+import { saveGeneratedFile } from "./generated-files.ts";
1011 import { isRecord } from "./history/shared.ts";
1112 import { isAnthropicModelFamily } from "./model-routing.ts";
1213 import {
@@ -370,6 +371,7 @@ export async function* mapFullStreamToTurnEvents(
370371 beforeTotalBytes: number;
371372 afterTotalBytes: number;
372373 }[];
374+ cwd?: string;
373375 },
374376 ): AsyncGenerator<TurnEvent> {
375377 const toolCallTracker = new StreamToolCallTracker();
@@ -382,6 +384,21 @@ export async function* mapFullStreamToTurnEvents(
382384 yield { type: "context-pruned", ...rec };
383385 }
384386 }
387+ // Handle file chunks from multimodal models (e.g. inline image generation)
388+ if (originalChunk.type === "file" && opts.cwd) {
389+ const fileData = originalChunk.file as
390+ | { mediaType: string; uint8Array: Uint8Array }
391+ | undefined;
392+ if (fileData?.uint8Array) {
393+ const filePath = await saveGeneratedFile(fileData, opts.cwd);
394+ yield {
395+ type: "file-generated",
396+ filePath,
397+ mediaType: fileData.mediaType,
398+ };
399+ continue;
400+ }
401+ }
385402 const prepared = toolCallTracker.prepare(originalChunk);
386403 const chunk = prepared.chunk;
387404 const route = textPhaseTracker.route(chunk);
src/llm-api/turn.ts+3−0
@@ -37,6 +37,7 @@ export async function* runTurn(options: {
3737 systemPrompt?: string;
3838 signal?: AbortSignal;
3939 thinkingEffort?: ThinkingEffort;
40+ cwd?: string;
4041 }): AsyncGenerator<TurnEvent> {
4142 const {
4243 model,
@@ -46,6 +47,7 @@ export async function* runTurn(options: {
4647 systemPrompt,
4748 signal,
4849 thinkingEffort,
50+ cwd,
4951 } = options;
5052
5153 const rawToolSet = buildToolSet(tools);
@@ -108,6 +110,7 @@ export async function* runTurn(options: {
108110
109111 for await (const event of mapFullStreamToTurnEvents(result.fullStream, {
110112 stepPruneQueue,
113+ ...(cwd ? { cwd } : {}),
111114 onChunk: (streamChunk) => {
112115 if (
113116 streamChunk.type === "tool-call" ||
src/llm-api/types.ts+7−0
@@ -52,6 +52,12 @@ export interface TurnCompleteEvent {
5252 messages: CoreMessage[];
5353 }
5454
55+export interface FileGeneratedEvent {
56+ type: "file-generated";
57+ filePath: string;
58+ mediaType: string;
59+}
60+
5561 export interface TurnErrorEvent {
5662 type: "turn-error";
5763 error: Error;
@@ -71,6 +77,7 @@ export type TurnEvent =
7177 | ToolCallStartEvent
7278 | ToolResultEvent
7379 | ContextPrunedEvent
80+ | FileGeneratedEvent
7481 | TurnCompleteEvent
7582 | TurnErrorEvent;