xonecas.io
64875
Mc

mini-coder / commit 524dc76

fix: per-step context pruning via prepareStep instead of middleware

The old wrapLanguageModel middleware operated on LanguageModelV2Prompt
(post-conversion), where consecutive tool messages are merged, skewing
the message count for pruneMessages. Replace with the SDK's prepareStep
hook which runs on CoreMessages before conversion, ensuring correct
pruning boundaries per step. Also prune coreHistory after each turn so
the next turn starts lean instead of re-pruning hundreds of accumulated
messages from scratch.

xonecas2026-03-20 524dc769efc32e94482f3f43ae6b5599ab77eb35 parent d713f17

3 files changed

src/agent/session-runner.ts+10−0
@@ -5,6 +5,10 @@ import type { ThinkingEffort } from "../llm-api/providers.ts";
55 import { resolveModel } from "../llm-api/providers.ts";
66 import type { CoreMessage } from "../llm-api/turn.ts";
77 import { runTurn } from "../llm-api/turn.ts";
8+import {
9+ applyContextPruning,
10+ compactToolResultPayloads,
11+} from "../llm-api/turn-context.ts";
812 import type { ToolDef } from "../llm-api/types.ts";
913 import { getMaxTurnIndex, saveMessages } from "../session/db/index.ts";
1014 import type { ActiveSession } from "../session/manager.ts";
@@ -227,6 +231,12 @@ export class SessionRunner {
227231 this.totalOut += outputTokens;
228232 this.lastContextTokens = contextTokens;
229233 touchActiveSession(this.session);
234+
235+ // Prune coreHistory after each turn so the next turn starts lean.
236+ // The DB retains the full history; this only affects in-memory context.
237+ this.coreHistory = compactToolResultPayloads(
238+ applyContextPruning(this.coreHistory),
239+ );
230240 } catch (err) {
231241 const stubMsg = makeInterruptMessage("error");
232242 this.coreHistory.push(stubMsg);
src/llm-api/turn-request.test.ts+193−17
@@ -2,26 +2,202 @@ import { describe, expect, test } from "bun:test";
22 import type { CoreMessage } from "./turn.ts";
33 import { buildStreamTextRequest } from "./turn-request.ts";
44
5+function makePrepared(messages: CoreMessage[]) {
6+ return {
7+ messages,
8+ systemPrompt: undefined,
9+ pruned: false,
10+ prePruneMessageCount: messages.length,
11+ postPruneMessageCount: messages.length,
12+ prePruneTotalBytes: 0,
13+ postPruneTotalBytes: 0,
14+ };
15+}
16+
17+function makeDefaultInput(
18+ messages: CoreMessage[],
19+ modelString = "openai/gpt-5.4",
20+) {
21+ return {
22+ model: {} as Parameters<typeof buildStreamTextRequest>[0]["model"],
23+ modelString,
24+ prepared: makePrepared(messages),
25+ toolSet: {},
26+ onStepFinish: () => {},
27+ signal: undefined,
28+ providerOptions: {},
29+ };
30+}
31+
32+/** Build a long history with tool-call + tool-result pairs. */
33+function buildToolHistory(pairCount: number): CoreMessage[] {
34+ const messages: CoreMessage[] = [];
35+ for (let i = 0; i < pairCount; i++) {
36+ messages.push({ role: "user", content: `u${i}` });
37+ messages.push({
38+ role: "assistant",
39+ content: [
40+ {
41+ type: "tool-call",
42+ toolCallId: `tc${i}`,
43+ toolName: "shell",
44+ input: {},
45+ },
46+ ],
47+ } as unknown as CoreMessage);
48+ messages.push({
49+ role: "tool",
50+ content: [
51+ {
52+ type: "tool-result",
53+ toolCallId: `tc${i}`,
54+ toolName: "shell",
55+ output: { text: `result-${i}` },
56+ },
57+ ],
58+ } as unknown as CoreMessage);
59+ }
60+ messages.push({ role: "user", content: "final" });
61+ return messages;
62+}
63+
64+interface PrepareStepResult {
65+ messages?: CoreMessage[];
66+}
67+
68+/** Call prepareStep on a request, failing if it's missing. */
69+function callPrepareStep(
70+ request: ReturnType<typeof buildStreamTextRequest>,
71+ stepNumber: number,
72+ messages: CoreMessage[],
73+): PrepareStepResult {
74+ const fn = request.prepareStep;
75+ if (!fn) throw new Error("prepareStep missing from request");
76+ return fn({
77+ stepNumber,
78+ messages,
79+ model: {} as never,
80+ steps: [],
81+ experimental_context: undefined,
82+ }) as PrepareStepResult;
83+}
84+
585 describe("buildStreamTextRequest", () => {
686 test("overrides the AI SDK default onError logger", () => {
7- const request = buildStreamTextRequest({
8- model: {} as Parameters<typeof buildStreamTextRequest>[0]["model"],
9- modelString: "openai/gpt-5.4",
10- prepared: {
11- messages: [{ role: "user", content: "hi" }] as CoreMessage[],
12- systemPrompt: undefined,
13- pruned: false,
14- prePruneMessageCount: 1,
15- postPruneMessageCount: 1,
16- prePruneTotalBytes: 2,
17- postPruneTotalBytes: 2,
18- },
19- toolSet: {},
20- onStepFinish: () => {},
21- signal: undefined,
22- providerOptions: {},
23- });
87+ const request = buildStreamTextRequest(
88+ makeDefaultInput([{ role: "user", content: "hi" }] as CoreMessage[]),
89+ );
2490
2591 expect(request.onError?.({ error: new Error("boom") })).toBeUndefined();
2692 });
93+
94+ test("prepareStep is present on the returned request", () => {
95+ const request = buildStreamTextRequest(
96+ makeDefaultInput([{ role: "user", content: "hi" }] as CoreMessage[]),
97+ );
98+ expect(request.prepareStep).toBeFunction();
99+ });
100+
101+ test("prepareStep skips pruning at step 0 (already pruned by prepareTurnMessages)", () => {
102+ const messages = buildToolHistory(30);
103+ const request = buildStreamTextRequest(makeDefaultInput(messages));
104+
105+ const result = callPrepareStep(request, 0, messages);
106+
107+ // For non-Anthropic models, step 0 returns empty (no modifications)
108+ expect(result).toEqual({});
109+ });
110+
111+ test("prepareStep prunes tool history at step 1+", () => {
112+ // 30 tool-call/result pairs = 91 messages (30*3 + 1 final user)
113+ const messages = buildToolHistory(30);
114+ const request = buildStreamTextRequest(makeDefaultInput(messages));
115+
116+ const result = callPrepareStep(request, 1, messages);
117+
118+ // pruneMessages with "before-last-40-messages" should remove old tool
119+ // history, resulting in fewer messages than the original 91
120+ expect(result.messages).toBeDefined();
121+ expect(result.messages?.length).toBeLessThan(messages.length);
122+ });
123+
124+ test("prepareStep does not prune short conversations at step 1+", () => {
125+ // 5 pairs = 16 messages, well under the 40-message threshold
126+ const messages = buildToolHistory(5);
127+ const request = buildStreamTextRequest(makeDefaultInput(messages));
128+
129+ const result = callPrepareStep(request, 1, messages);
130+
131+ // Short conversations should keep all messages
132+ expect(result.messages).toBeDefined();
133+ expect(result.messages?.length).toBe(messages.length);
134+ });
135+
136+ test("prepareStep adds cache breakpoints for Anthropic models", () => {
137+ const messages: CoreMessage[] = [
138+ { role: "user", content: "hello" },
139+ { role: "assistant", content: "hi" } as CoreMessage,
140+ ];
141+ const request = buildStreamTextRequest(
142+ makeDefaultInput(messages, "anthropic/claude-sonnet-4-5"),
143+ );
144+
145+ const result = callPrepareStep(request, 0, messages);
146+
147+ // Should return messages with cache breakpoint annotations
148+ expect(result.messages).toBeDefined();
149+ const annotated = result.messages ?? [];
150+ const withCache = annotated.filter(
151+ (m: CoreMessage) =>
152+ m.providerOptions?.anthropic &&
153+ (m.providerOptions.anthropic as Record<string, unknown>).cacheControl,
154+ );
155+ expect(withCache.length).toBeGreaterThan(0);
156+ });
157+
158+ test("prepareStep prunes consistently across multiple steps", () => {
159+ // Simulate growing context across steps
160+ const base = buildToolHistory(30); // 91 messages
161+ const request = buildStreamTextRequest(makeDefaultInput(base));
162+
163+ // Step 1: prune from 91 messages
164+ const result1 = callPrepareStep(request, 1, base);
165+
166+ // Step 5: add more messages (simulating accumulated tool calls)
167+ const grown = [
168+ ...base,
169+ {
170+ role: "assistant",
171+ content: [
172+ {
173+ type: "tool-call",
174+ toolCallId: "extra1",
175+ toolName: "shell",
176+ input: {},
177+ },
178+ ],
179+ } as unknown as CoreMessage,
180+ {
181+ role: "tool",
182+ content: [
183+ {
184+ type: "tool-result",
185+ toolCallId: "extra1",
186+ toolName: "shell",
187+ output: { text: "extra" },
188+ },
189+ ],
190+ } as unknown as CoreMessage,
191+ ];
192+ const result5 = callPrepareStep(request, 5, grown);
193+
194+ // Both should prune, and the larger input should not produce a much
195+ // larger output — pruning keeps the tail window roughly constant
196+ const len1 = result1.messages?.length ?? 0;
197+ const len5 = result5.messages?.length ?? 0;
198+ expect(len1).toBeLessThan(base.length);
199+ expect(len5).toBeLessThan(grown.length);
200+ // The pruned sizes should be similar (within a small window)
201+ expect(Math.abs(len1 - len5)).toBeLessThanOrEqual(4);
202+ });
27203 });
src/llm-api/turn-request.ts+25−22
@@ -1,4 +1,4 @@
1-import { type streamText, wrapLanguageModel } from "ai";
1+import type { streamText } from "ai";
22
33 import { isAnthropicModelFamily } from "./model-routing.ts";
44 import type { ThinkingEffort } from "./provider-options.ts";
@@ -59,35 +59,38 @@ interface BuildStreamTextRequestInput {
5959 export function buildStreamTextRequest(
6060 input: BuildStreamTextRequestInput,
6161 ): StreamTextOptions {
62- // Wrap with per-step middleware that prunes context and compacts payloads
63- // between multi-step tool-use rounds. For Anthropic models we also annotate
64- // cache breakpoints so they track the moving conversation tail.
62+ // Per-step context management via prepareStep: prunes old tool history,
63+ // compacts large payloads, and annotates Anthropic cache breakpoints.
64+ // Runs on CoreMessages before SDK conversion, so pruneMessages operates
65+ // on the correct types and message boundaries.
6566 const isAnthropic = isAnthropicModelFamily(input.modelString);
66- const model = wrapLanguageModel({
67- model: input.model as Parameters<typeof wrapLanguageModel>[0]["model"],
68- middleware: [
69- {
70- specificationVersion: "v3" as const,
71- transformParams: async ({ params }) => {
72- const prompt = params.prompt as CoreMessage[];
73- const pruned = applyContextPruning(prompt);
74- const compacted = compactToolResultPayloads(pruned);
75- const final = isAnthropic
76- ? annotateAnthropicCacheBreakpoints(compacted)
77- : compacted;
78- return { ...params, prompt: final as typeof params.prompt };
79- },
80- },
81- ],
82- });
8367
8468 return {
85- model,
69+ model: input.model,
8670 maxOutputTokens: 16384,
8771 messages: input.prepared.messages,
8872 tools: input.toolSet,
8973 stopWhen: continueUntilModelStops,
9074 onStepFinish: input.onStepFinish,
75+ prepareStep: ({ stepNumber, messages }) => {
76+ // Step 0 messages are already pruned by prepareTurnMessages;
77+ // only apply cache breakpoints.
78+ if (stepNumber === 0) {
79+ return isAnthropic
80+ ? {
81+ messages: annotateAnthropicCacheBreakpoints(
82+ messages as CoreMessage[],
83+ ) as typeof messages,
84+ }
85+ : {};
86+ }
87+ const pruned = applyContextPruning(messages as CoreMessage[]);
88+ const compacted = compactToolResultPayloads(pruned);
89+ const final = isAnthropic
90+ ? annotateAnthropicCacheBreakpoints(compacted)
91+ : compacted;
92+ return { messages: final as typeof messages };
93+ },
9194 ...(input.prepared.systemPrompt
9295 ? { system: input.prepared.systemPrompt }
9396 : {}),