xonecas.io
64875
Mc

mini-coder / commit f0816dc

chore: add terminal-bench benchmark workspace

xonecas2026-04-09 f0816dcdbf9b54fdae688671d7b3955b7a33c637 parent ccb7c49

4 files changed

.gitignore+2−0
@@ -12,3 +12,5 @@ package-lock.json
1212 logs/
1313 .codex
1414 .claude
15+terminal-bench/jobs/
16+terminal-bench/__pycache__/
AGENTS.md+1−0
@@ -9,6 +9,7 @@
99
1010 - Read `spec.md`, then `TODO.md`, then inspect `git status --short` / `git diff`.
1111 - Treat `TODO.md` as the current verified work queue. For spec-alignment work, confirm the mismatch in code before editing.
12+- For audits, treat open `TODO.md` items as already-known issues and focus findings on new gaps unless the user asks to re-check known debt.
1213 - Keep `TODO.md` minimal and current; remove completed items instead of accumulating history.
1314
1415 ## Code map
terminal-bench/mini_coder_agent.py+116−0
@@ -0,0 +1,116 @@
1+from __future__ import annotations
2+
3+import shlex
4+from pathlib import Path
5+
6+from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template
7+from harbor.environments.base import BaseEnvironment
8+from harbor.models.agent.context import AgentContext
9+from harbor.models.trial.paths import EnvironmentPaths
10+
11+
12+class MiniCoderAgent(BaseInstalledAgent):
13+ """Harbor installed-agent wrapper for the published mini-coder CLI."""
14+
15+ def __init__(self, *args, version: str = "0.5.0", **kwargs):
16+ super().__init__(*args, version=version, **kwargs)
17+ self._host_config_dir = Path.home() / ".config" / "mini-coder"
18+ self._host_settings_path = self._host_config_dir / "settings.json"
19+ self._host_auth_path = self._host_config_dir / "auth.json"
20+
21+ @staticmethod
22+ def name() -> str:
23+ return "mini-coder"
24+
25+ async def install(self, environment: BaseEnvironment) -> None:
26+ await self.exec_as_root(
27+ environment,
28+ command=(
29+ "if command -v apt-get >/dev/null 2>&1; then "
30+ "apt-get update && apt-get install -y curl unzip ca-certificates bash git; "
31+ "elif command -v apk >/dev/null 2>&1; then "
32+ "apk add --no-cache curl unzip ca-certificates bash git; "
33+ "else "
34+ 'echo "Unsupported package manager for mini-coder install" >&2; exit 1; '
35+ "fi"
36+ ),
37+ env={"DEBIAN_FRONTEND": "noninteractive"},
38+ )
39+ await self.exec_as_agent(
40+ environment,
41+ command=(
42+ "set -euo pipefail; "
43+ "curl -fsSL https://bun.sh/install | bash; "
44+ 'export PATH="$HOME/.bun/bin:$PATH"; '
45+ f"bun add -g mini-coder@{self._version}; "
46+ "test -x ~/.bun/bin/bun; "
47+ "test -x ~/.bun/bin/mc"
48+ ),
49+ )
50+
51+ def populate_context_post_run(self, context: AgentContext) -> None:
52+ return None
53+
54+ def _require_host_config(self) -> None:
55+ missing: list[str] = []
56+ if not self._host_settings_path.is_file():
57+ missing.append(str(self._host_settings_path))
58+ if not self._host_auth_path.is_file():
59+ missing.append(str(self._host_auth_path))
60+ if missing:
61+ raise FileNotFoundError(
62+ "Missing local mini-coder config files required for the Harbor run: "
63+ + ", ".join(missing)
64+ )
65+
66+ async def _container_config_dir(self, environment: BaseEnvironment) -> str:
67+ result = await self.exec_as_agent(environment, command='printf %s "$HOME"')
68+ home_dir = (result.stdout or "").strip()
69+ if not home_dir:
70+ raise RuntimeError("Failed to determine the agent home directory")
71+ config_dir = f"{home_dir}/.config/mini-coder"
72+ await self.exec_as_agent(
73+ environment,
74+ command=f"mkdir -p {shlex.quote(config_dir)}",
75+ )
76+ return config_dir
77+
78+ async def _upload_host_config(
79+ self, environment: BaseEnvironment, config_dir: str
80+ ) -> None:
81+ settings_target = f"{config_dir}/settings.json"
82+ auth_target = f"{config_dir}/auth.json"
83+ await environment.upload_file(self._host_settings_path, settings_target)
84+ await environment.upload_file(self._host_auth_path, auth_target)
85+ await self.exec_as_agent(
86+ environment,
87+ command=(
88+ f"chmod 600 {shlex.quote(settings_target)} {shlex.quote(auth_target)}"
89+ ),
90+ )
91+
92+ @with_prompt_template
93+ async def run(
94+ self,
95+ instruction: str,
96+ environment: BaseEnvironment,
97+ context: AgentContext,
98+ ) -> None:
99+ del context
100+ self._require_host_config()
101+ config_dir = await self._container_config_dir(environment)
102+ await self._upload_host_config(environment, config_dir)
103+
104+ escaped_instruction = shlex.quote(instruction)
105+ ndjson_path = (EnvironmentPaths.agent_dir / "mini-coder.ndjson").as_posix()
106+ stderr_path = (EnvironmentPaths.agent_dir / "mini-coder.stderr.txt").as_posix()
107+
108+ await self.exec_as_agent(
109+ environment,
110+ command=(
111+ 'export PATH="$HOME/.bun/bin:$PATH"; '
112+ f"mc -p {escaped_instruction} </dev/null "
113+ f">{shlex.quote(ndjson_path)} "
114+ f"2>{shlex.quote(stderr_path)}"
115+ ),
116+ )
terminal-bench/readme.md+165−0
@@ -0,0 +1,165 @@
1+# terminal-bench benchmark workspace
2+
3+This folder is the scratch space for benchmarking mini-coder against Terminal-Bench 2.0 with Harbor.
4+
5+Everything here is intentionally kept under `terminal-bench/` so it does it's separate from production code.
6+
7+## Goal
8+
9+Benchmark published `mini-coder@latest` on Terminal-Bench 2.0 using Harbor, with the current local mini-coder config:
10+
11+- model: `openai-codex/gpt-5.4`
12+- effort: `xhigh`
13+- auth source: `~/.config/mini-coder/auth.json`
14+- settings source: `~/.config/mini-coder/settings.json`
15+
16+## Key decisions
17+
18+1. **Use Harbor's custom installed-agent path**
19+ - Harbor does not require multiple models.
20+ - We use a custom installed agent instead of changing mini-coder core behavior.
21+
22+2. **Benchmark published mini-coder, not the local checkout**
23+ - The wrapper installs `mini-coder@latest` inside the task container.
24+ - This was the chosen path instead of packaging the current repo state.
25+
26+3. **Use the current mini-coder OAuth config**
27+ - `openai-codex` in mini-coder is OAuth-backed.
28+ - The wrapper copies the local `auth.json` and `settings.json` into the container.
29+ - It uploads copies only; it does **not** mount or mutate the host files.
30+
31+4. **Keep the integration outside the product code**
32+ - The Harbor agent lives here in `mini_coder_agent.py`.
33+ - No tracked repo files were changed for the benchmark setup.
34+
35+## Files
36+
37+- `mini_coder_agent.py` — Harbor installed-agent wrapper for mini-coder
38+- `jobs/` — Harbor job outputs from smoke runs
39+
40+## Current wrapper behavior
41+
42+`mini_coder_agent.py` does this per trial:
43+
44+1. installs system deps: `curl`, `unzip`, `ca-certificates`, `bash`, `git`
45+2. installs Bun
46+3. installs published `mini-coder@latest`
47+4. finds the container user's home directory
48+5. uploads copies of:
49+ - `~/.config/mini-coder/settings.json`
50+ - `~/.config/mini-coder/auth.json`
51+6. runs:
52+ - `mc -p <task instruction>`
53+7. writes logs to:
54+ - `/logs/agent/mini-coder.ndjson`
55+ - `/logs/agent/mini-coder.stderr.txt`
56+
57+## Progress so far
58+
59+### Smoke run 1
60+
61+Command:
62+
63+```bash
64+PYTHONPATH=$PWD/terminal-bench harbor run -y \
65+ -d terminal-bench@2.0 \
66+ --agent-import-path mini_coder_agent:MiniCoderAgent \
67+ -m openai-codex/gpt-5.4 \
68+ -l 1 -k 1 -n 1 \
69+ -o terminal-bench/jobs
70+```
71+
72+Job dir:
73+
74+- `jobs/2026-04-09__22-14-57`
75+
76+Result:
77+
78+- failed immediately before the agent really started
79+- root cause: `git` was missing in the task container
80+- mini-coder stderr contained:
81+
82+```text
83+Executable not found in $PATH: "git"
84+```
85+
86+Fix applied afterward:
87+
88+- updated the wrapper install step to include `git`
89+
90+### Smoke run 2
91+
92+Same command after the `git` fix.
93+
94+Job dir:
95+
96+- `jobs/2026-04-09__22-17-21`
97+
98+Result:
99+
100+- the run got past startup
101+- mini-coder emitted headless NDJSON thinking output
102+- Harbor job was interrupted/cancelled before completion
103+- Harbor recorded a `CancelledError`
104+
105+Evidence:
106+
107+- `jobs/2026-04-09__22-17-21/*/agent/mini-coder.ndjson` contains streamed model output
108+- `jobs/2026-04-09__22-17-21/*/exception.txt` shows cancellation
109+
110+## Current status
111+
112+The wrapper is past the first blocker (`git` missing) and is capable of launching mini-coder inside the Harbor task container.
113+
114+The next real checkpoint is to rerun the same 1-task smoke test and let it finish.
115+
116+## Resume commands
117+
118+### Verify the wrapper still imports
119+
120+```bash
121+PYTHONPATH=$PWD/terminal-bench \
122+ ~/.local/share/pipx/venvs/harbor/bin/python -m py_compile \
123+ terminal-bench/mini_coder_agent.py
124+```
125+
126+### Run the 1-task smoke test again
127+
128+```bash
129+PYTHONPATH=$PWD/terminal-bench harbor run -y \
130+ -d terminal-bench@2.0 \
131+ --agent-import-path mini_coder_agent:MiniCoderAgent \
132+ -m openai-codex/gpt-5.4 \
133+ -l 1 -k 1 -n 1 \
134+ -o terminal-bench/jobs
135+```
136+
137+### Inspect the latest Harbor job
138+
139+```bash
140+JOB=$(ls -dt terminal-bench/jobs/* | head -n1)
141+find "$JOB" -maxdepth 3 -type f | sort
142+```
143+
144+### Inspect mini-coder logs from the latest trial
145+
146+```bash
147+JOB=$(ls -dt terminal-bench/jobs/* | head -n1)
148+sed -n '1,120p' "$JOB"/*/agent/mini-coder.stderr.txt
149+sed -n '1,120p' "$JOB"/*/agent/mini-coder.ndjson
150+```
151+
152+## Known constraints / notes
153+
154+- This setup depends on the host mini-coder OAuth/login state being current.
155+- If `openai-codex` auth expires, refresh it locally with mini-coder before rerunning.
156+- Because the wrapper copies the host config into the container, the host auth/settings files are not modified by Harbor.
157+- The first two smoke jobs are kept intentionally for debugging history.
158+
159+## Next step
160+
161+Rerun the same 1-task smoke test and let it complete. If that passes the launch phase cleanly, decide whether to:
162+
163+1. keep iterating with `-l 1 -k 1`
164+2. run a small subset
165+3. run a larger TB2 batch later