xonecas.io
64875
Mc

mini-coder / commit fa9b964

refactor: dedupe UTF-8 truncation logic

xonecas2026-04-08 fa9b964277153d9fccae0fa7318f80ec8a5069de parent 40f0799

2 files changed

src/tools.test.ts+36โˆ’0
@@ -47,6 +47,31 @@ function readFile(name: string): string {
4747 return readFileSync(join(tmp, name), "utf-8");
4848 }
4949
50+function hasUnpairedSurrogate(input: string): boolean {
51+ for (let index = 0; index < input.length; index++) {
52+ const codeUnit = input.charCodeAt(index);
53+ const isHighSurrogate = codeUnit >= 0xd800 && codeUnit <= 0xdbff;
54+ const isLowSurrogate = codeUnit >= 0xdc00 && codeUnit <= 0xdfff;
55+
56+ if (isHighSurrogate) {
57+ const nextCodeUnit = input.charCodeAt(index + 1);
58+ const nextIsLowSurrogate =
59+ nextCodeUnit >= 0xdc00 && nextCodeUnit <= 0xdfff;
60+ if (!nextIsLowSurrogate) {
61+ return true;
62+ }
63+ index++;
64+ continue;
65+ }
66+
67+ if (isLowSurrogate) {
68+ return true;
69+ }
70+ }
71+
72+ return false;
73+}
74+
5075 // ---------------------------------------------------------------------------
5176 // edit
5277 // ---------------------------------------------------------------------------
@@ -383,6 +408,17 @@ describe("truncateOutput", () => {
383408 expect(Buffer.byteLength(result, "utf8")).toBeLessThanOrEqual(2_000);
384409 });
385410
411+ test("preserves surrogate pairs when truncating by UTF-8 byte size", () => {
412+ const input = `${"๐Ÿ™‚".repeat(6)}ABC${"๐Ÿ™ƒ".repeat(6)}`;
413+ const result = truncateOutput(input, 100, 42);
414+
415+ expect(result).toContain("โ€ฆ truncated for size โ€ฆ");
416+ expect(result.startsWith("๐Ÿ™‚")).toBe(true);
417+ expect(result.endsWith("๐Ÿ™ƒ")).toBe(true);
418+ expect(hasUnpairedSurrogate(result)).toBe(false);
419+ expect(Buffer.byteLength(result, "utf8")).toBeLessThanOrEqual(42);
420+ });
421+
386422 test("head and tail do not overlap", () => {
387423 const lines = Array.from({ length: 50 }, (_, i) => `L${i + 1}`);
388424 const input = lines.join("\n");
src/tools.ts+68โˆ’24
@@ -295,42 +295,85 @@ function buildLineTruncation(
295295 };
296296 }
297297
298-/** Slice the largest UTF-8 prefix that fits within `maxBytes`. */
299-function sliceUtf8Prefix(input: string, maxBytes: number): string {
298+function isHighSurrogate(codeUnit: number): boolean {
299+ return codeUnit >= 0xd800 && codeUnit <= 0xdbff;
300+}
301+
302+function isLowSurrogate(codeUnit: number): boolean {
303+ return codeUnit >= 0xdc00 && codeUnit <= 0xdfff;
304+}
305+
306+function findUtf8SliceLength(
307+ input: string,
308+ maxBytes: number,
309+ getCandidate: (length: number) => string,
310+): number {
300311 if (maxBytes <= 0 || input === "") {
301- return "";
312+ return 0;
302313 }
314+
303315 let low = 0;
304316 let high = input.length;
305317 while (low < high) {
306318 const mid = Math.ceil((low + high) / 2);
307- const candidate = input.slice(0, mid);
319+ const candidate = getCandidate(mid);
308320 if (Buffer.byteLength(candidate, "utf8") <= maxBytes) {
309321 low = mid;
310322 } else {
311323 high = mid - 1;
312324 }
313325 }
314- return input.slice(0, low);
326+
327+ return low;
315328 }
316329
317-/** Slice the largest UTF-8 suffix that fits within `maxBytes`. */
318-function sliceUtf8Suffix(input: string, maxBytes: number): string {
319- if (maxBytes <= 0 || input === "") {
320- return "";
330+function normalizeUtf8PrefixEnd(input: string, end: number): number {
331+ if (end <= 0 || end >= input.length) {
332+ return end;
321333 }
322- let low = 0;
323- let high = input.length;
324- while (low < high) {
325- const mid = Math.ceil((low + high) / 2);
326- const candidate = input.slice(input.length - mid);
327- if (Buffer.byteLength(candidate, "utf8") <= maxBytes) {
328- low = mid;
329- } else {
330- high = mid - 1;
331- }
334+
335+ const previousCodeUnit = input.charCodeAt(end - 1);
336+ const nextCodeUnit = input.charCodeAt(end);
337+ if (isHighSurrogate(previousCodeUnit) && isLowSurrogate(nextCodeUnit)) {
338+ return end - 1;
339+ }
340+
341+ return end;
342+}
343+
344+function normalizeUtf8SuffixStart(input: string, start: number): number {
345+ if (start <= 0 || start >= input.length) {
346+ return start;
347+ }
348+
349+ const previousCodeUnit = input.charCodeAt(start - 1);
350+ const nextCodeUnit = input.charCodeAt(start);
351+ if (isHighSurrogate(previousCodeUnit) && isLowSurrogate(nextCodeUnit)) {
352+ return start + 1;
332353 }
333- return input.slice(input.length - low);
354+
355+ return start;
356+}
357+
358+/** Slice the largest UTF-8 prefix that fits within `maxBytes`. */
359+function sliceUtf8Prefix(input: string, maxBytes: number): string {
360+ const end = normalizeUtf8PrefixEnd(
361+ input,
362+ findUtf8SliceLength(input, maxBytes, (length) => input.slice(0, length)),
363+ );
364+ return input.slice(0, end);
365+}
366+
367+/** Slice the largest UTF-8 suffix that fits within `maxBytes`. */
368+function sliceUtf8Suffix(input: string, maxBytes: number): string {
369+ const start = normalizeUtf8SuffixStart(
370+ input,
371+ input.length -
372+ findUtf8SliceLength(input, maxBytes, (length) =>
373+ input.slice(input.length - length),
374+ ),
375+ );
376+ return input.slice(start);
334377 }
335378
336379 /** Fit disjoint head/tail segments plus a marker within a UTF-8 byte budget. */
@@ -357,17 +400,18 @@ function fitSegmentsWithinBytes(
357400 let remainingBytes = availableBytes - usedBytes;
358401
359402 if (remainingBytes > 0) {
403+ const headBytes = Buffer.byteLength(head, "utf8");
360404 const expandedHead = sliceUtf8Prefix(
361405 headSource,
362- headBudget + remainingBytes,
406+ headBytes + remainingBytes,
363407 );
364- remainingBytes -=
365- Buffer.byteLength(expandedHead, "utf8") - Buffer.byteLength(head, "utf8");
408+ remainingBytes -= Buffer.byteLength(expandedHead, "utf8") - headBytes;
366409 head = expandedHead;
367410 }
368411
369412 if (remainingBytes > 0) {
370- tail = sliceUtf8Suffix(tailSource, tailBudget + remainingBytes);
413+ const tailBytes = Buffer.byteLength(tail, "utf8");
414+ tail = sliceUtf8Suffix(tailSource, tailBytes + remainingBytes);
371415 }
372416
373417 return head + marker + tail;