xonecas.io
8447
Co

conveyor / files

conveyor / tools / bake.py

12.9 KB
1#!/usr/bin/env python3
2"""Bakes conveyor's base VM image from Debian's cloud image for this host's architecture, with QEMU
3run as the user. conveyord runs every ticket in an overlay on this image. Run it again only to change
4the image: delete images/base.qcow2 first.
5
6    tools/bake.py bake
7
8State lives in ~/.local/share/conveyor. Sockets live in /tmp/cv-*, because unix socket paths max out at 104 bytes.
9"""
10import http.server, json, os, platform, signal, socket, subprocess, sys, threading, time, urllib.request
11from pathlib import Path
12
13DATA = Path.home() / ".local/share/conveyor"
14REPO = Path(__file__).resolve().parent.parent
15BASE = DATA / "images/base.qcow2"
16KEY = DATA / "id_ed25519"
17ARM = platform.machine() in ("arm64", "aarch64")
18CLOUD = DATA / f"images/debian-13-{'arm64' if ARM else 'amd64'}.qcow2"
19CLOUD_URL = f"https://cloud.debian.org/images/cloud/trixie/latest/{CLOUD.name.replace('debian-13', 'debian-13-genericcloud')}"
20USER, PASS = "sacenox", "zzzz"
21DEV_PKGS = """starship fzf tree-sitter-cli rustup nodejs npm golang-go python3 python3-pip xdotool wmctrl maim x11-utils
22kitty neovim git curl build-essential libxkbcommon-dev qemu-guest-agent mesa-vulkan-drivers fonts-jetbrains-mono""".split()
23XORG_PKGS = "libwnck-3-0 mesa-utils xserver-xorg xinit xinput x11-xserver-utils".split()
24DE_PKGS = """blueman file-roller galculator gvfs lightdm lightdm-gtk-greeter ristretto thunar-archive-plugin
25xdg-user-dirs-gtk mousepad xfce4 xfce4-notifyd xfce4-pulseaudio-plugin xfce4-screenshooter xfce4-taskmanager
26xfce4-whiskermenu-plugin""".split()
27
28
29def sock(name, kind):
30    return f"/tmp/cv-{name}.{kind}"
31
32
33def log(*a):
34    print(f"[{time.strftime('%H:%M:%S')}]", *a, flush=True)
35
36
37# ---------- QMP: JSON over a unix socket ----------
38
39class Channel:
40    def __init__(self, path, timeout=10):
41        self.s = socket.socket(socket.AF_UNIX)
42        self.s.settimeout(timeout)
43        self.s.connect(path)
44        self.buf = b""
45
46    def call(self, execute, **arguments):
47        msg = {"execute": execute, **({"arguments": arguments} if arguments else {})}
48        self.s.sendall(json.dumps(msg).encode() + b"\n")
49        while True:
50            while b"\n" not in self.buf:
51                chunk = self.s.recv(65536)
52                if not chunk:
53                    raise ConnectionError("channel closed")
54                self.buf += chunk
55            line, self.buf = self.buf.split(b"\n", 1)
56            r = json.loads(line)
57            if "return" in r or "error" in r:
58                if "error" in r:
59                    raise RuntimeError(f"{execute}: {r['error']}")
60                return r["return"]
61
62    def close(self):
63        self.s.close()
64
65
66def qmp(name):
67    c = Channel(sock(name, "qmp"))
68    c.buf = b""
69    while b"\n" not in c.buf:
70        c.buf += c.s.recv(65536)  # greeting
71    c.buf = c.buf.split(b"\n", 1)[1]
72    c.call("qmp_capabilities")
73    return c
74
75
76# ---------- QEMU ----------
77
78def ssh_port(name):
79    return 22000 + (sum(name.encode()) % 1000)
80
81
82def boot(name, disk, mem_mb=8192, cpus=8, seed=None):
83    for k in ("qmp", "qga", "vnc", "pid"):
84        Path(sock(name, k)).unlink(missing_ok=True)
85    args = [
86        f"qemu-system-{'aarch64' if ARM else 'x86_64'}", "-name", name, "-accel", "hvf" if sys.platform == "darwin" else "kvm",
87        "-machine", "virt" if ARM else "q35", "-cpu", "host", "-smp", str(cpus), "-m", str(mem_mb),
88        *(["-bios", "edk2-aarch64-code.fd"] if ARM else []),
89        "-drive", f"file={disk},if=virtio,format=qcow2,discard=unmap",
90        "-netdev", f"user,id=n0,hostfwd=tcp:127.0.0.1:{ssh_port(name)}-:22", "-device", "virtio-net-pci,netdev=n0",
91        "-device", "virtio-balloon-pci,free-page-reporting=on",
92        "-device", "virtio-serial-pci",
93        "-chardev", f"socket,path={sock(name, 'qga')},server=on,wait=off,id=qga0",
94        "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
95        "-qmp", f"unix:{sock(name, 'qmp')},server=on,wait=off",
96        *(["-device", "virtio-gpu-pci"] if ARM else ["-vga", "virtio"]), "-display", "none", "-vnc", f"unix:{sock(name, 'vnc')}",
97        "-device", "qemu-xhci", "-device", "usb-tablet",
98        "-pidfile", sock(name, "pid"), "-daemonize",
99    ]
100    if seed:
101        args += ["-smbios", f"type=1,serial=ds=nocloud;s={seed}"]
102    subprocess.run(args, check=True)
103
104
105def ssh(name, *cmd, check=True, timeout=None):
106    return subprocess.run(["ssh", "-i", str(KEY), "-p", str(ssh_port(name)), "-o", "BatchMode=yes",
107                           "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null",
108                           "-o", "LogLevel=ERROR", "-o", "ConnectTimeout=5", f"{USER}@127.0.0.1", *cmd],
109                          check=check, timeout=timeout, capture_output=not sys.stdout.isatty(), text=True)
110
111
112def wait_ssh(name, secs=300):
113    end = time.monotonic() + secs
114    while time.monotonic() < end:
115        try:
116            if ssh(name, "true", check=False, timeout=8).returncode == 0:
117                return True
118        except subprocess.TimeoutExpired:
119            pass
120        time.sleep(0.5)
121    return False
122
123
124def poweroff(name, secs=120):
125    try:
126        qmp(name).call("system_powerdown")
127    except (OSError, RuntimeError):
128        pass
129    pid_file = Path(sock(name, "pid"))
130    end = time.monotonic() + secs
131    while pid_file.exists() and time.monotonic() < end:
132        pid = int(pid_file.read_text() or 0)
133        if not pid or subprocess.call(["kill", "-0", str(pid)], stderr=subprocess.DEVNULL):
134            break
135        time.sleep(1)
136    else:
137        if pid_file.exists():
138            try:
139                qmp(name).call("quit")
140            except OSError:
141                pass
142    for k in ("qmp", "qga", "vnc", "pid"):
143        Path(sock(name, k)).unlink(missing_ok=True)
144
145
146# ---------- bake ----------
147
148PROVISION = f"""#!/bin/bash
149set -euo pipefail
150export DEBIAN_FRONTEND=noninteractive
151apt-get update
152apt-get -y full-upgrade
153apt-get install -y {' '.join(DEV_PKGS + DE_PKGS + XORG_PKGS)}
154# The cloud kernel has no GPU driver, and so no desktop: swap it for the generic one.
155apt-get install -y linux-image-$(dpkg --print-architecture)
156apt-get purge -y linux-image-cloud-$(dpkg --print-architecture) 'linux-image-*-cloud-*'
157systemctl enable lightdm.service qemu-guest-agent.service
158systemctl set-default graphical.target
159install -d /etc/lightdm/lightdm.conf.d
160printf '%s\\n' '[Seat:*]' 'autologin-user={USER}' 'autologin-user-timeout=0' 'autologin-session=xfce' > /etc/lightdm/lightdm.conf.d/50-autologin.conf
161systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target
162git clone --depth 1 https://github.com/sacenox/dotfiles /tmp/dotfiles
163cp -a /tmp/dotfiles/. /home/{USER}/
164rm -rf /home/{USER}/.git /home/{USER}/README.md
165install -d /etc/X11/xorg.conf.d /home/{USER}/.config/autostart /home/{USER}/.config/xfce4/xfconf/xfce-perchannel-xml
166printf '%s\\n' 'Section "ServerFlags"' '    Option "BlankTime" "0"' '    Option "StandbyTime" "0"' \\
167    '    Option "SuspendTime" "0"' '    Option "OffTime" "0"' EndSection > /etc/X11/xorg.conf.d/10-noblank.conf
168printf '%s\\n' '[Desktop Entry]' Hidden=true > /home/{USER}/.config/autostart/xfce4-screensaver.desktop
169cat > /home/{USER}/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml <<'XML'
170<?xml version="1.0" encoding="UTF-8"?>
171<channel name="xfce4-power-manager" version="1.0">
172  <property name="xfce4-power-manager" type="empty"><property name="dpms-enabled" type="bool" value="false"/></property>
173</channel>
174XML
175chown -R {USER}:{USER} /home/{USER}
176runuser -u {USER} -- env HOME=/home/{USER} npm config set prefix /home/{USER}/.local
177runuser -u {USER} -- env HOME=/home/{USER} bash -c 'curl -fsSL https://claude.ai/install.sh | bash'
178# A restored snapshot can leave a stale forwarded socket behind; let a new session replace it.
179printf 'StreamLocalBindUnlink yes\\n' > /etc/ssh/sshd_config.d/50-conveyor.conf
180# Every overlay gets its own ssh host keys at first boot: Debian's sshd won't start without them.
181install -d /etc/systemd/system/ssh.service.d
182printf '%s\\n' '[Service]' 'ExecStartPre=' 'ExecStartPre=/usr/bin/ssh-keygen -A' 'ExecStartPre=/usr/sbin/sshd -t' > /etc/systemd/system/ssh.service.d/keygen.conf
183# The seed is gone after the bake: cloud-init must not run again.
184touch /etc/cloud/cloud-init.disabled
185"""
186
187
188def serve_seed():
189    """cloud-init's NoCloud seed over HTTP: the guest reaches the host's loopback at 10.0.2.2."""
190    files = {"/meta-data": b"instance-id: conveyor-base\nlocal-hostname: conveyor-base\n", "/vendor-data": b"",
191             "/user-data": b"#cloud-config\n" + json.dumps({
192                 "users": [{"name": USER, "groups": ["sudo"], "shell": "/bin/bash", "lock_passwd": False,
193                            "plain_text_passwd": PASS, "ssh_authorized_keys": [KEY.with_suffix(".pub").read_text().strip()]}],
194                 "write_files": [{"path": "/root/provision.sh", "permissions": "0700", "content": PROVISION}],
195                 "runcmd": [["sh", "-c", "/root/provision.sh > /var/log/provision.log 2>&1 && touch /var/log/provision.ok"]],
196             }).encode()}
197
198    class Seed(http.server.BaseHTTPRequestHandler):
199        def do_GET(self):
200            body = files.get(self.path)
201            self.send_response(200 if body is not None else 404)
202            self.end_headers()
203            self.wfile.write(body or b"")
204
205        def log_message(self, *a):
206            pass
207
208    srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Seed)
209    threading.Thread(target=srv.serve_forever, daemon=True).start()
210    return f"http://10.0.2.2:{srv.server_port}/"
211
212
213def bake():
214    if BASE.exists():
215        sys.exit(f"{BASE} exists. Delete it to bake again.")
216    BASE.parent.mkdir(parents=True, exist_ok=True)
217    if not CLOUD.exists():
218        log(f"downloading {CLOUD_URL}")
219        urllib.request.urlretrieve(CLOUD_URL, CLOUD.with_suffix(".part"))
220        CLOUD.with_suffix(".part").rename(CLOUD)
221    if not KEY.exists():
222        subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "conveyor", "-f", str(KEY)], check=True)
223    tmp = BASE.with_suffix(".baking.qcow2")
224    subprocess.run(["qemu-img", "convert", "-q", "-O", "qcow2", str(CLOUD), str(tmp)], check=True)
225    subprocess.run(["qemu-img", "resize", "-q", str(tmp), "60G"], check=True)
226    t0 = time.monotonic()
227    log(f"booting {CLOUD.name}")
228    boot("bake", tmp, seed=serve_seed())
229    if not wait_ssh("bake", 300):
230        sys.exit("the cloud image does not answer ssh")
231    log(f"up after {time.monotonic() - t0:.0f} s; provisioning")
232    r = ssh("bake", "cloud-init status --wait >/dev/null; tail -n 40 /var/log/provision.log; test -e /var/log/provision.ok", check=False)
233    if r.returncode:
234        sys.exit(f"provisioning failed:\n{r.stdout or ''}")
235    log(f"provisioned after {time.monotonic() - t0:.0f} s; rebooting into the desktop")
236    poweroff("bake")
237    boot("bake", tmp)
238    if not wait_ssh("bake", 300):
239        sys.exit("baked image does not answer ssh")
240    r = ssh("bake", "systemctl is-system-running --wait; ~/.local/bin/claude --version; dpkg-query -W xfce4-session qemu-guest-agent; "
241                    "pgrep -u sacenox xfce4-session", check=False)
242    log("verify:", (r.stdout or "").strip().replace("\n", " | "))
243    # Rust and a warm cargo cache, so a widget ticket builds in seconds, not minutes.
244    log("pre-building the widget library")
245    # In ~/work, where agents build: cargo fingerprints workspace crates by path.
246    # Without macOS metadata: its ._* files would read as widgets.
247    tar = subprocess.run(["tar", "--no-xattrs", "-C", str(REPO / "templates"), "--exclude=library/target", "-cf", "-", "library"],
248                         capture_output=True, check=True, env={**os.environ, "COPYFILE_DISABLE": "1"}).stdout
249    subprocess.run(["ssh", "-i", str(KEY), "-p", str(ssh_port("bake")), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no",
250                    "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", f"{USER}@127.0.0.1", "tar -xf - -C /tmp && mv /tmp/library ~/work"], input=tar, check=True)
251    r = ssh("bake", "rustup default 1.99.1 >/dev/null 2>&1 && cd ~/work && CARGO_TARGET_DIR=$HOME/.cache/conveyor-target "
252                    "cargo build --release -p preview 2>&1 | tail -1; cd && rm -rf ~/work", check=False)
253    log("library:", (r.stdout or r.stderr or "").strip())
254    # Reset what must be unique per overlay, then shut down clean.
255    ssh("bake", f"echo {PASS} | sudo -S sh -c 'rm -f /etc/ssh/ssh_host_* && : > /etc/machine-id && fstrim -a'", check=False)
256    poweroff("bake")
257    subprocess.run(["qemu-img", "convert", "-O", "qcow2", "-q", str(tmp), str(BASE)], check=True)
258    tmp.unlink()
259    BASE.chmod(0o444)
260    log(f"base ready: {BASE} ({BASE.stat().st_size / 2**30:.1f} GiB) in {time.monotonic() - t0:.0f} s")
261
262
263if __name__ == "__main__":
264    if sys.argv[1:] != ["bake"]:
265        sys.exit(print(__doc__))
266    # A killed or closed bake exits like Ctrl+C, so the VM below never outlives it.
267    for s in (signal.SIGTERM, signal.SIGHUP):
268        signal.signal(s, lambda n, _: sys.exit(128 + n))
269    try:
270        bake()
271    finally:
272        poweroff("bake", 0)