@@ -81,6 +81,23 @@ function makeTodoSnapshotMessage(
8181 };
8282 }
8383
84+function makeImageToolResultMessage(index: number): Message {
85+ return {
86+ role: "toolResult",
87+ toolCallId: `image-${index}`,
88+ toolName: "readImage",
89+ content: [
90+ {
91+ type: "image",
92+ data: Buffer.from(`image-${index}`).toString("base64"),
93+ mimeType: "image/png",
94+ },
95+ ],
96+ isError: false,
97+ timestamp: Date.now() + index,
98+ };
99+}
100+
84101 const ZERO_USAGE: Usage = {
85102 input: 0,
86103 output: 0,
@@ -1873,6 +1890,194 @@ describe("agent loop", () => {
18731890 expect(assistantMessages).toHaveLength(3);
18741891 });
18751892
1893+ test("older image tool results are replaced once the context image budget is exceeded", async () => {
1894+ const api = "test/image-budget";
1895+ const sourceId = "test-image-budget";
1896+ const model: Model<string> = {
1897+ id: api,
1898+ name: "Image Budget Model",
1899+ api,
1900+ provider: "test",
1901+ baseUrl: "http://localhost:0",
1902+ reasoning: false,
1903+ input: ["text", "image"],
1904+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
1905+ contextWindow: 128000,
1906+ maxTokens: 4096,
1907+ };
1908+ const requestMessages: Message[][] = [];
1909+
1910+ const buildProviderStream = (context: Context) => {
1911+ requestMessages.push(structuredClone(context.messages));
1912+ const stream = createAssistantMessageEventStream();
1913+ const finalMessage = fauxAssistantMessage("done");
1914+
1915+ queueMicrotask(() => {
1916+ stream.push({ type: "done", reason: "stop", message: finalMessage });
1917+ stream.end(finalMessage);
1918+ });
1919+
1920+ return stream;
1921+ };
1922+
1923+ registerApiProvider(
1924+ {
1925+ api,
1926+ stream: (_model, context) => buildProviderStream(context),
1927+ streamSimple: (_model, context) => buildProviderStream(context),
1928+ },
1929+ sourceId,
1930+ );
1931+
1932+ try {
1933+ const session = createSession(db, { cwd: tmp });
1934+ const userMsg = makeUser("describe the screenshots");
1935+ const turn = appendMessage(db, session.id, userMsg);
1936+ const messages: Message[] = [
1937+ userMsg,
1938+ ...Array.from({ length: 14 }, (_, index) =>
1939+ makeImageToolResultMessage(index + 1),
1940+ ),
1941+ ];
1942+
1943+ await runAgentLoop({
1944+ db,
1945+ sessionId: session.id,
1946+ turn,
1947+ model,
1948+ systemPrompt: "Test",
1949+ tools: [],
1950+ toolHandlers: new Map(),
1951+ messages,
1952+ cwd: tmp,
1953+ });
1954+
1955+ expect(requestMessages).toHaveLength(1);
1956+ const contextMessages = requestMessages[0]!;
1957+ const imageBlocks = contextMessages.flatMap((message) =>
1958+ Array.isArray(message.content)
1959+ ? message.content.filter((block) => block.type === "image")
1960+ : [],
1961+ );
1962+ expect(imageBlocks).toHaveLength(12);
1963+
1964+ const firstImageResult = contextMessages[1];
1965+ const secondImageResult = contextMessages[2];
1966+ if (
1967+ !firstImageResult ||
1968+ firstImageResult.role !== "toolResult" ||
1969+ !Array.isArray(firstImageResult.content)
1970+ ) {
1971+ throw new Error("Expected the first image tool result in context");
1972+ }
1973+ if (
1974+ !secondImageResult ||
1975+ secondImageResult.role !== "toolResult" ||
1976+ !Array.isArray(secondImageResult.content)
1977+ ) {
1978+ throw new Error("Expected the second image tool result in context");
1979+ }
1980+
1981+ expect(firstImageResult.content).toEqual([
1982+ {
1983+ type: "text",
1984+ text: "[Earlier image omitted from model context to stay within image-input limits.]",
1985+ },
1986+ ]);
1987+ expect(secondImageResult.content).toEqual(firstImageResult.content);
1988+
1989+ const newestImageResult = contextMessages.at(-1);
1990+ if (
1991+ !newestImageResult ||
1992+ newestImageResult.role !== "toolResult" ||
1993+ !Array.isArray(newestImageResult.content)
1994+ ) {
1995+ throw new Error("Expected the newest image tool result in context");
1996+ }
1997+ expect(newestImageResult.content[0]?.type).toBe("image");
1998+ } finally {
1999+ unregisterApiProviders(sourceId);
2000+ }
2001+ });
2002+
2003+ test("image blocks are replaced in model context for text-only models", async () => {
2004+ const api = "test/text-only-image-context";
2005+ const sourceId = "test-text-only-image-context";
2006+ const model: Model<string> = {
2007+ id: api,
2008+ name: "Text Only Context Model",
2009+ api,
2010+ provider: "test",
2011+ baseUrl: "http://localhost:0",
2012+ reasoning: false,
2013+ input: ["text"],
2014+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
2015+ contextWindow: 128000,
2016+ maxTokens: 4096,
2017+ };
2018+ const requestMessages: Message[][] = [];
2019+
2020+ const buildProviderStream = (context: Context) => {
2021+ requestMessages.push(structuredClone(context.messages));
2022+ const stream = createAssistantMessageEventStream();
2023+ const finalMessage = fauxAssistantMessage("done");
2024+
2025+ queueMicrotask(() => {
2026+ stream.push({ type: "done", reason: "stop", message: finalMessage });
2027+ stream.end(finalMessage);
2028+ });
2029+
2030+ return stream;
2031+ };
2032+
2033+ registerApiProvider(
2034+ {
2035+ api,
2036+ stream: (_model, context) => buildProviderStream(context),
2037+ streamSimple: (_model, context) => buildProviderStream(context),
2038+ },
2039+ sourceId,
2040+ );
2041+
2042+ try {
2043+ const session = createSession(db, { cwd: tmp });
2044+ const userMsg = makeUser("describe the screenshot");
2045+ const turn = appendMessage(db, session.id, userMsg);
2046+ const messages: Message[] = [userMsg, makeImageToolResultMessage(1)];
2047+
2048+ await runAgentLoop({
2049+ db,
2050+ sessionId: session.id,
2051+ turn,
2052+ model,
2053+ systemPrompt: "Test",
2054+ tools: [],
2055+ toolHandlers: new Map(),
2056+ messages,
2057+ cwd: tmp,
2058+ });
2059+
2060+ expect(requestMessages).toHaveLength(1);
2061+ const imageResult = requestMessages[0]?.[1];
2062+ if (
2063+ !imageResult ||
2064+ imageResult.role !== "toolResult" ||
2065+ !Array.isArray(imageResult.content)
2066+ ) {
2067+ throw new Error("Expected the image tool result in context");
2068+ }
2069+
2070+ expect(imageResult.content).toEqual([
2071+ {
2072+ type: "text",
2073+ text: "[Image omitted from model context because the active model does not support image input.]",
Cut short: this commit's diff is over 1000 lines.