xonecas.io
7517
Cd

clanker-and-the-dev / files

clanker-and-the-dev / .agents / skills / add-new-comic / scripts / generate_panel.py

35.8 KB
1from __future__ import annotations
2
3import random
4import re
5from pathlib import Path
6
7from PIL import Image, ImageDraw, ImageOps
8
9from models import ComicSpec, GenerationConfig, PanelSpec
10
11_PIPELINE = None
12_PIPELINE_KEY = None
13
14STYLE_PROMPT = (
15    "single plain wordless comic illustration, clean green line sketch on solid black background, "
16    "thin unfilled strokes, simple Dev, boxy Clanker, comic art only"
17)
18
19# Normalized crop boxes for art-reference/training-reference.png.  The sheet is
20# intentionally clean and black-backed; cropping lets IP-Adapter see large
21# character examples instead of a tiny downscaled full character sheet.
22TRAINING_REFERENCE_CROPS: dict[str, tuple[float, float, float, float]] = {
23    "clanker_neutral_body": (0.013, 0.026, 0.101, 0.271),
24    "clanker_happy_body": (0.106, 0.026, 0.193, 0.271),
25    "clanker_side_body": (0.382, 0.026, 0.430, 0.270),
26    "clanker_at_laptop": (0.720, 0.043, 0.859, 0.272),
27    "clanker_neutral_face": (0.019, 0.297, 0.100, 0.492),
28    "clanker_happy_face": (0.119, 0.298, 0.210, 0.492),
29    "clanker_thinking_face": (0.322, 0.300, 0.403, 0.492),
30    "clanker_concerned_face": (0.423, 0.299, 0.503, 0.492),
31    "clanker_overheated_face": (0.632, 0.300, 0.727, 0.492),
32    "clanker_heart_face": (0.749, 0.299, 0.829, 0.492),
33    "dev_neutral_body": (0.024, 0.523, 0.080, 0.729),
34    "dev_mug_body": (0.105, 0.523, 0.161, 0.729),
35    "dev_laptop_body": (0.286, 0.536, 0.370, 0.727),
36    "dev_seated_laptop": (0.387, 0.537, 0.508, 0.728),
37    "dev_thumbs_body": (0.544, 0.523, 0.618, 0.726),
38    "dev_excited_body": (0.796, 0.525, 0.885, 0.723),
39    "dev_running_body": (0.896, 0.526, 0.989, 0.723),
40    "dev_neutral_face": (0.020, 0.775, 0.084, 0.915),
41    "dev_happy_face": (0.106, 0.775, 0.170, 0.915),
42    "dev_confused_face": (0.193, 0.776, 0.257, 0.915),
43    "dev_frown_face": (0.279, 0.776, 0.343, 0.914),
44    "dev_panic_face": (0.443, 0.768, 0.537, 0.914),
45    "dev_love_face": (0.752, 0.762, 0.906, 0.917),
46}
47
48# Normalized crop boxes for art-reference/ip-character-reference.png.  The
49# source file is a good high-quality style anchor, but the full image contains
50# two large black character cards and baseline shadows that SDXL tends to copy
51# as stray background panels.  Auto-cropping feeds just the characters to
52# IP-Adapter while keeping the high-resolution proportions/style signal.
53IP_CHARACTER_REFERENCE_CROPS: dict[str, tuple[float, float, float, float]] = {
54    "ip_dev_full": (0.080, 0.085, 0.270, 0.555),
55    "ip_clanker_full": (0.385, 0.088, 0.620, 0.560),
56}
57
58BASE_REFERENCE_CROPS = [
59    "dev_neutral_body",
60    "clanker_neutral_body",
61    "dev_laptop_body",
62    "clanker_happy_body",
63]
64
65# Extra pose crops used only for auto ControlNet guides. These are cut from
66# the clean art-reference sheets so the guide itself has the same proportions
67# as the target comic instead of a generic stick-figure sketch.
68CONTROL_REFERENCE_SPRITES: dict[str, tuple[str, tuple[float, float, float, float]]] = {
69    **{name: ("training-reference.png", box) for name, box in TRAINING_REFERENCE_CROPS.items()},
70    "dev_pose_overwhelmed": ("dev-poses-sheet.png", (0.845, 0.285, 0.940, 0.470)),
71    "dev_pose_collapsed": ("dev-poses-sheet.png", (0.345, 0.590, 0.485, 0.775)),
72    "dev_pose_dead": ("dev-poses-sheet.png", (0.505, 0.570, 0.640, 0.775)),
73    "dev_pose_slumped": ("dev-poses-sheet.png", (0.170, 0.600, 0.325, 0.755)),
74    "dev_pose_zombie": ("dev-poses-sheet.png", (0.655, 0.600, 0.790, 0.790)),
75}
76
77_CONTROL_SPRITE_CACHE: dict[tuple[str, str], Image.Image] = {}
78
79
80def _torch_dtype(torch, dtype_name: str):
81    return {
82        "float16": torch.float16,
83        "bfloat16": torch.bfloat16,
84        "float32": torch.float32,
85    }[dtype_name]
86
87
88def _pipeline_key(config: GenerationConfig) -> tuple[object, ...]:
89    return (
90        config.modelId,
91        config.enableControlNet,
92        config.controlNetModelId,
93        config.ipAdapterRepo,
94        config.ipAdapterSubfolder,
95        config.ipAdapterWeight,
96        config.ipAdapterScale,
97        config.device,
98        config.torchDtype,
99    )
100
101
102def load_pipeline(config: GenerationConfig):
103    global _PIPELINE, _PIPELINE_KEY
104    key = _pipeline_key(config)
105    if _PIPELINE is not None and _PIPELINE_KEY == key:
106        return _PIPELINE
107
108    import torch
109    from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, StableDiffusionXLPipeline
110
111    if config.device == "cuda" and not torch.cuda.is_available():
112        raise RuntimeError("CUDA was requested but torch.cuda.is_available() is false")
113
114    dtype = _torch_dtype(torch, config.torchDtype)
115    variant = "fp16" if config.torchDtype == "float16" else None
116
117    if config.enableControlNet:
118        controlnet = ControlNetModel.from_pretrained(
119            config.controlNetModelId,
120            torch_dtype=dtype,
121            use_safetensors=True,
122        )
123        pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
124            config.modelId,
125            controlnet=controlnet,
126            torch_dtype=dtype,
127            use_safetensors=True,
128            variant=variant,
129        )
130    else:
131        pipe = StableDiffusionXLPipeline.from_pretrained(
132            config.modelId,
133            torch_dtype=dtype,
134            use_safetensors=True,
135            variant=variant,
136        )
137
138    pipe.load_ip_adapter(
139        config.ipAdapterRepo,
140        subfolder=config.ipAdapterSubfolder,
141        weight_name=config.ipAdapterWeight,
142    )
143    pipe.set_ip_adapter_scale(config.ipAdapterScale)
144
145    if config.device == "cuda" and config.enableSequentialCpuOffload and hasattr(pipe, "enable_sequential_cpu_offload"):
146        # SDXL + IP-Adapter + ControlNet does not fit reliably in 12 GB with
147        # model-level offload. Sequential offload is slower but keeps the full
148        # quality path usable on the target RTX 3080 Ti.
149        pipe.enable_sequential_cpu_offload()
150    elif config.device == "cuda" and config.enableModelCpuOffload:
151        pipe.enable_model_cpu_offload()
152    else:
153        pipe.to(config.device)
154
155    if config.enableVaeSlicing and hasattr(pipe, "enable_vae_slicing"):
156        pipe.enable_vae_slicing()
157    if config.enableVaeTiling and hasattr(pipe, "enable_vae_tiling"):
158        pipe.enable_vae_tiling()
159    if config.enableAttentionSlicing and hasattr(pipe, "enable_attention_slicing"):
160        pipe.enable_attention_slicing()
161
162    _PIPELINE = pipe
163    _PIPELINE_KEY = key
164    return pipe
165
166
167def release_pipeline(device: str = "cuda") -> None:
168    global _PIPELINE, _PIPELINE_KEY
169    if _PIPELINE is None:
170        return
171
172    pipe = _PIPELINE
173    _PIPELINE = None
174    _PIPELINE_KEY = None
175
176    try:
177        if hasattr(pipe, "maybe_free_model_hooks"):
178            pipe.maybe_free_model_hooks()
179    except Exception:
180        pass
181    try:
182        pipe.to("cpu")
183    except Exception:
184        pass
185
186    del pipe
187
188    import gc
189
190    gc.collect()
191    if device == "cuda":
192        try:
193            import torch
194
195            torch.cuda.empty_cache()
196            torch.cuda.ipc_collect()
197        except Exception:
198            pass
199
200
201def compact_visual_text(text: str) -> str:
202    replacements = {
203        "Dev, a human software developer with simple rectangular glasses": "Dev",
204        "a human software developer with simple rectangular glasses": "Dev",
205        "Clanker, a helpful boxy robot assistant with screen eyes and antenna": "Clanker",
206        "Clanker, a boxy robot assistant with screen eyes and antenna": "Clanker",
207        "a helpful boxy robot assistant with screen eyes and antenna": "Clanker",
208        "a boxy robot assistant with screen eyes and antenna": "Clanker",
209        "pure black terminal room": "plain black room",
210        "pure black terminal workspace": "plain black workspace",
211        "dark neon terminal room": "plain black room",
212        "black neon terminal room": "plain black room",
213        "black terminal room": "plain black room",
214        "dark neon terminal workspace": "plain black workspace",
215        "black neon terminal workspace": "plain black workspace",
216        "black terminal workspace": "plain black workspace",
217        "green screen glow": "terminal lines",
218        "small green suggestion glow": "small cursor",
219        "boot-up glow": "boot-up cursor",
220        "terminal boot glow": "boot cursor",
221        "glowing laptop": "laptop",
222        "glowing": "terminal",
223        "glows": "shows",
224        "glow": "terminal line",
225        "in the same neon terminal room with a confident robot grin on its screen eyes while": "while",
226        "in a plain black room while": "while",
227        "in a black terminal room while": "while",
228        "sparks and tiny green debris implying the source code vanished": "sparks and debris",
229        "sparse dark upper area reserved for speech bubbles": "empty dark top",
230        "dark upper area reserved for a speech bubble": "empty dark top",
231        "empty dark upper area": "empty top",
232        "empty top area": "empty top",
233        "terminal line energy": "simple motion lines",
234        "terminal cursor": "small cursor",
235        "terminal laptop": "laptop",
236        "Dev on the left": "Dev left",
237        "Clanker on the right": "Clanker right",
238        "on the left at the laptop": "left at laptop",
239        "on the right in a triumphant pose": "right triumphant",
240        "on the left": "left",
241        "on the right": "right",
242        "overwhelmed Dev on the left": "Dev left panicked",
243        "laptop screen centered": "laptop center",
244        "leans in helpfully with one claw raised": "stands helpfully",
245        "laptop foreground": "laptop",
246        "foreground": "",
247    }
248    for old, new in replacements.items():
249        text = text.replace(old, new)
250    text = re.sub(r"\b(Dev|Clanker),\s*", r"\1 ", text)
251    return re.sub(r"\s+", " ", text).strip()
252
253
254def limit_words(text: str, max_words: int) -> str:
255    words = text.split()
256    return " ".join(words[:max_words])
257
258
259def build_prompt(comic: ComicSpec, panel: PanelSpec, panel_number: int) -> str:
260    # CLIP token limits are tight. Put the house style first, then a compact
261    # scene description; the reference-derived ControlNet guide carries layout.
262    parts = [
263        STYLE_PROMPT,
264        f"{panel.shot} shot, full bodies",
265        limit_words(compact_visual_text(panel.artPrompt), 22),
266    ]
267    return ". ".join(part.strip().rstrip(".") for part in parts if part and part.strip()) + "."
268
269
270def _panel_text(panel: PanelSpec) -> str:
271    dialogue = " ".join(line.text for line in panel.dialogue)
272    return " ".join(
273        part
274        for part in [panel.artPrompt, panel.composition or "", panel.caption or "", dialogue]
275        if part
276    ).lower()
277
278
279def _has_any(text: str, words: tuple[str, ...]) -> bool:
280    return any(word in text for word in words)
281
282
283def reference_crop_names_for_panel(panel: PanelSpec) -> list[str]:
284    text = _panel_text(panel)
285    names = list(BASE_REFERENCE_CROPS)
286
287    if _has_any(text, ("laptop", "typing", "terminal", "desk", "seated", "nvim")):
288        names.extend(["dev_seated_laptop", "clanker_at_laptop"])
289    if _has_any(text, ("excited", "wow", "hype", "hyped", "celebrat", "triumphant", "proud", "confident")):
290        names.extend(["dev_excited_body", "dev_happy_face", "clanker_happy_face"])
291    if _has_any(text, ("panic", "panicked", "overwhelmed", "chaos", "sweat", "holy", "sparks", "overclock")):
292        names.extend(["dev_panic_face", "clanker_overheated_face"])
293    if _has_any(text, ("dead", "collapsed", "floor", "burnout", "flat", "concerned")):
294        names.extend(["dev_frown_face", "clanker_concerned_face"])
295    if _has_any(text, ("thinking", "confused", "curious", "question")):
296        names.extend(["dev_confused_face", "clanker_thinking_face"])
297    if _has_any(text, ("coffee", "mug")):
298        names.append("dev_mug_body")
299    if _has_any(text, ("running", "run")):
300        names.append("dev_running_body")
301    if _has_any(text, ("love", "heart")):
302        names.extend(["dev_love_face", "clanker_heart_face"])
303
304    # Keep order stable but remove duplicates.
305    return list(dict.fromkeys(names))
306
307
308def square_on_black(image: Image.Image, size: int = 512) -> Image.Image:
309    image = ImageOps.exif_transpose(image).convert("RGB")
310    image = ImageOps.contain(image, (size, size), method=Image.Resampling.LANCZOS)
311    canvas = Image.new("RGB", (size, size), "#020806")
312    canvas.paste(image, ((size - image.width) // 2, (size - image.height) // 2))
313    return canvas
314
315
316def crop_normalized(image: Image.Image, box: tuple[float, float, float, float]) -> Image.Image:
317    width, height = image.size
318    left, top, right, bottom = box
319    crop_box = (
320        max(0, round(left * width)),
321        max(0, round(top * height)),
322        min(width, round(right * width)),
323        min(height, round(bottom * height)),
324    )
325    return image.crop(crop_box)
326
327
328def _expanded_bbox(box: tuple[int, int, int, int], size: tuple[int, int], padding: int = 10) -> tuple[int, int, int, int]:
329    left, top, right, bottom = box
330    width, height = size
331    return (
332        max(0, left - padding),
333        max(0, top - padding),
334        min(width, right + padding),
335        min(height, bottom + padding),
336    )
337
338
339def control_reference_sprite(project_root: Path, name: str) -> Image.Image | None:
340    spec = CONTROL_REFERENCE_SPRITES.get(name)
341    if spec is None:
342        return None
343
344    cache_key = (str(project_root.resolve()), name)
345    cached = _CONTROL_SPRITE_CACHE.get(cache_key)
346    if cached is not None:
347        return cached.copy()
348
349    source_name, box = spec
350    path = project_root / "art-reference" / source_name
351    if not path.exists():
352        return None
353
354    crop = crop_normalized(Image.open(path).convert("RGB"), box)
355    alpha = ImageOps.grayscale(crop).point(lambda p: 0 if p < 28 else min(255, p * 2))
356    bbox = alpha.getbbox()
357    if bbox is None:
358        return None
359
360    bbox = _expanded_bbox(bbox, crop.size, padding=12)
361    alpha = alpha.crop(bbox)
362    sprite = Image.new("RGBA", alpha.size, (255, 255, 255, 0))
363    sprite.putalpha(alpha)
364    _CONTROL_SPRITE_CACHE[cache_key] = sprite
365    return sprite.copy()
366
367
368def paste_control_sprite(
369    image: Image.Image,
370    sprite: Image.Image,
371    center_x: int,
372    baseline_y: int,
373    target_height: int,
374) -> None:
375    if target_height <= 0:
376        return
377    scale = target_height / sprite.height
378    target_width = max(1, round(sprite.width * scale))
379    sprite = sprite.resize((target_width, target_height), Image.Resampling.LANCZOS)
380    x = round(center_x - target_width / 2)
381    y = round(baseline_y - target_height)
382    image.paste(sprite, (x, y), sprite)
383
384
385def training_reference_crops(image: Image.Image, panel: PanelSpec) -> list[Image.Image]:
386    crops: list[Image.Image] = []
387    for name in reference_crop_names_for_panel(panel):
388        box = TRAINING_REFERENCE_CROPS.get(name)
389        if box is not None:
390            crops.append(square_on_black(crop_normalized(image, box)))
391    return crops
392
393
394def ip_character_reference_crops(image: Image.Image) -> list[Image.Image]:
395    return [square_on_black(crop_normalized(image, box)) for box in IP_CHARACTER_REFERENCE_CROPS.values()]
396
397
398def load_reference_images(config: GenerationConfig, panel: PanelSpec, project_root: Path) -> list[Image.Image]:
399    images: list[Image.Image] = []
400    for raw_path in config.referenceImages:
401        path = (project_root / raw_path).resolve()
402        if not path.exists():
403            raise FileNotFoundError(f"Reference image not found: {path}")
404        image = Image.open(path).convert("RGB")
405        if config.referenceCropMode == "auto" and path.name == "training-reference.png":
406            images.extend(training_reference_crops(image, panel))
407        elif config.referenceCropMode == "auto" and path.name == "ip-character-reference.png":
408            images.extend(ip_character_reference_crops(image))
409        else:
410            images.append(square_on_black(image))
411
412    if not images:
413        raise ValueError("At least one reference image is required for IP-Adapter conditioning")
414    return images
415
416
417def make_reference_contact_sheet(images: list[Image.Image], size: int = 384) -> Image.Image:
418    tiles = [square_on_black(image, size=size) for image in images]
419    cols = 1 if len(tiles) == 1 else 2
420    rows = (len(tiles) + cols - 1) // cols
421    sheet = Image.new("RGB", (cols * size, rows * size), "#020806")
422    for index, image in enumerate(tiles):
423        x = (index % cols) * size
424        y = (index // cols) * size
425        sheet.paste(image, (x, y))
426    return sheet
427
428
429def make_ip_adapter_image_embeds(pipe, reference_images: list[Image.Image], torch, device, do_classifier_free_guidance: bool):
430    projection = getattr(getattr(pipe.unet, "encoder_hid_proj", None), "image_projection_layers", None)
431    if not projection or len(projection) != 1:
432        return None
433
434    from diffusers.models.embeddings import ImageProjection
435
436    output_hidden_state = not isinstance(projection[0], ImageProjection)
437    positive_embeds = []
438    negative_embeds = []
439    with torch.no_grad():
440        for image in reference_images:
441            positive, negative = pipe.encode_image(image, device, 1, output_hidden_state)
442            positive_embeds.append(positive.detach().cpu())
443            negative_embeds.append(negative.detach().cpu())
444            if torch.cuda.is_available():
445                torch.cuda.empty_cache()
446
447    positive = torch.stack(positive_embeds, dim=0).mean(dim=0)
448    negative = torch.stack(negative_embeds, dim=0).mean(dim=0)
449
450    # Keep the CLIP image encoder from occupying VRAM during ControlNet/UNet
451    # denoising. Diffusers' model CPU offload does not always evict it after a
452    # manual encode_image call.
453    if getattr(pipe, "image_encoder", None) is not None:
454        try:
455            pipe.image_encoder.to("cpu")
456        except Exception:
457            pass
458    if torch.cuda.is_available():
459        torch.cuda.empty_cache()
460
461    # The original IP-Adapter projection expects [batch, image_count, embed_dim].
462    # Hidden-state variants already include the token dimension.
463    if not output_hidden_state:
464        positive = positive.unsqueeze(1)
465        negative = negative.unsqueeze(1)
466
467    if do_classifier_free_guidance:
468        return [torch.cat([negative, positive], dim=0)]
469    return [positive]
470
471
472def _clause_position(text: str, name: str, default: float) -> float:
473    clauses = re.split(r"[.;,]", text)
474    for clause in clauses:
475        if name not in clause:
476            continue
477        if "left" in clause:
478            return 0.30
479        if "right" in clause:
480            return 0.70
481        if "center" in clause or "middle" in clause:
482            return 0.50
483        if "foreground" in clause:
484            return 0.36 if name == "dev" else 0.64
485    return default
486
487
488def draw_laptop(draw: ImageDraw.ImageDraw, width: int, height: int, x: float, y: float, scale: float, stroke: int):
489    screen_w = int(170 * scale)
490    screen_h = int(100 * scale)
491    keyboard_h = int(34 * scale)
492    x0 = int(x - screen_w / 2)
493    y0 = int(y - screen_h)
494    x1 = int(x + screen_w / 2)
495    y1 = int(y)
496    draw.rectangle((x0, y0, x1, y1), outline="white", width=stroke)
497    draw.line((x0 + stroke, y1, x1 - stroke, y1), fill="white", width=stroke)
498    draw.polygon(
499        [
500            (x0 - int(24 * scale), y1 + keyboard_h),
501            (x1 + int(24 * scale), y1 + keyboard_h),
502            (x1, y1),
503            (x0, y1),
504        ],
505        outline="white",
506    )
507    draw.line((x0 + int(46 * scale), y0 + int(45 * scale), x1 - int(46 * scale), y0 + int(45 * scale)), fill="white", width=stroke)
508    draw.line((x0 + int(70 * scale), y0 + int(28 * scale), x0 + int(45 * scale), y0 + int(50 * scale)), fill="white", width=stroke)
509    draw.line((x1 - int(70 * scale), y0 + int(28 * scale), x1 - int(45 * scale), y0 + int(50 * scale)), fill="white", width=stroke)
510
511
512def draw_token_meter(draw: ImageDraw.ImageDraw, width: int, height: int, stroke: int):
513    x0 = int(width * 0.24)
514    y0 = int(height * 0.82)
515    x1 = int(width * 0.76)
516    y1 = int(height * 0.90)
517    draw.rounded_rectangle((x0, y0, x1, y1), radius=10, outline="white", width=stroke)
518    segments = 12
519    gap = 4
520    seg_w = (x1 - x0 - 40 - gap * (segments - 1)) // segments
521    sx = x0 + 20
522    for _ in range(segments):
523        draw.rectangle((sx, y0 + 20, sx + seg_w, y1 - 20), outline="white", width=max(1, stroke // 2))
524        sx += seg_w + gap
525
526
527def draw_dev(draw: ImageDraw.ImageDraw, x: int, base_y: int, scale: float, state: str, stroke: int):
528    if state == "collapsed":
529        head_r = int(38 * scale)
530        head_x = x - int(52 * scale)
531        head_y = base_y - int(35 * scale)
532        draw.ellipse((head_x - head_r, head_y - head_r, head_x + head_r, head_y + head_r), outline="white", width=stroke)
533        draw.rectangle((head_x - int(30 * scale), head_y - int(10 * scale), head_x - int(4 * scale), head_y + int(8 * scale)), outline="white", width=max(1, stroke // 2))
534        draw.rectangle((head_x + int(4 * scale), head_y - int(10 * scale), head_x + int(30 * scale), head_y + int(8 * scale)), outline="white", width=max(1, stroke // 2))
535        draw.line((head_x + head_r, head_y, x + int(95 * scale), base_y), fill="white", width=stroke)
536        draw.line((x + int(20 * scale), base_y, x + int(110 * scale), base_y + int(35 * scale)), fill="white", width=stroke)
537        draw.line((x + int(20 * scale), base_y, x + int(110 * scale), base_y - int(35 * scale)), fill="white", width=stroke)
538        return
539
540    seated = state == "seated"
541    head_r = int(38 * scale)
542    head_y = base_y - int(170 * scale if seated else 210 * scale)
543    draw.ellipse((x - head_r, head_y - head_r, x + head_r, head_y + head_r), outline="white", width=stroke)
544    draw.rectangle((x - int(34 * scale), head_y - int(10 * scale), x - int(5 * scale), head_y + int(9 * scale)), outline="white", width=max(1, stroke // 2))
545    draw.rectangle((x + int(5 * scale), head_y - int(10 * scale), x + int(34 * scale), head_y + int(9 * scale)), outline="white", width=max(1, stroke // 2))
546    draw.line((x - int(5 * scale), head_y, x + int(5 * scale), head_y), fill="white", width=max(1, stroke // 2))
547
548    neck_y = head_y + head_r
549    hip_y = base_y - int(62 * scale if seated else 70 * scale)
550    draw.line((x, neck_y, x, hip_y), fill="white", width=stroke)
551
552    if state == "excited":
553        draw.line((x, neck_y + int(18 * scale), x - int(58 * scale), neck_y - int(50 * scale)), fill="white", width=stroke)
554        draw.line((x, neck_y + int(18 * scale), x + int(58 * scale), neck_y - int(50 * scale)), fill="white", width=stroke)
555    elif seated:
556        draw.line((x, neck_y + int(25 * scale), x + int(80 * scale), neck_y + int(60 * scale)), fill="white", width=stroke)
557        draw.line((x, neck_y + int(25 * scale), x + int(58 * scale), neck_y + int(82 * scale)), fill="white", width=stroke)
558    else:
559        draw.line((x, neck_y + int(25 * scale), x - int(45 * scale), neck_y + int(88 * scale)), fill="white", width=stroke)
560        draw.line((x, neck_y + int(25 * scale), x + int(45 * scale), neck_y + int(88 * scale)), fill="white", width=stroke)
561
562    if seated:
563        draw.line((x, hip_y, x + int(70 * scale), base_y - int(34 * scale)), fill="white", width=stroke)
564        draw.line((x + int(70 * scale), base_y - int(34 * scale), x + int(105 * scale), base_y - int(34 * scale)), fill="white", width=stroke)
565        draw.line((x, hip_y, x - int(15 * scale), base_y), fill="white", width=stroke)
566        draw.line((x - int(50 * scale), base_y, x + int(35 * scale), base_y), fill="white", width=stroke)
567    else:
568        draw.line((x, hip_y, x - int(42 * scale), base_y), fill="white", width=stroke)
569        draw.line((x, hip_y, x + int(42 * scale), base_y), fill="white", width=stroke)
570        draw.line((x - int(42 * scale), base_y, x - int(65 * scale), base_y), fill="white", width=stroke)
571        draw.line((x + int(42 * scale), base_y, x + int(65 * scale), base_y), fill="white", width=stroke)
572
573
574def draw_clanker(draw: ImageDraw.ImageDraw, x: int, base_y: int, scale: float, state: str, stroke: int):
575    head_w = int(110 * scale)
576    head_h = int(82 * scale)
577    head_y0 = base_y - int(255 * scale)
578    head_y1 = head_y0 + head_h
579    draw.rounded_rectangle((x - head_w // 2, head_y0, x + head_w // 2, head_y1), radius=int(10 * scale), outline="white", width=stroke)
580    draw.rectangle((x - int(28 * scale), head_y0 + int(30 * scale), x - int(14 * scale), head_y0 + int(55 * scale)), outline="white", width=stroke)
581    draw.rectangle((x + int(14 * scale), head_y0 + int(30 * scale), x + int(28 * scale), head_y0 + int(55 * scale)), outline="white", width=stroke)
582    if state == "worried":
583        draw.arc((x - int(20 * scale), head_y0 + int(54 * scale), x + int(20 * scale), head_y0 + int(82 * scale)), 190, 350, fill="white", width=stroke)
584    else:
585        draw.line((x - int(20 * scale), head_y0 + int(68 * scale), x + int(20 * scale), head_y0 + int(68 * scale)), fill="white", width=stroke)
586    draw.line((x, head_y0, x, head_y0 - int(42 * scale)), fill="white", width=stroke)
587    draw.ellipse((x - int(8 * scale), head_y0 - int(58 * scale), x + int(8 * scale), head_y0 - int(42 * scale)), outline="white", width=stroke)
588
589    body_y0 = head_y1 + int(18 * scale)
590    body_y1 = body_y0 + int(90 * scale)
591    draw.rounded_rectangle((x - int(42 * scale), body_y0, x + int(42 * scale), body_y1), radius=int(8 * scale), outline="white", width=stroke)
592    draw.line((x - int(20 * scale), body_y0 + int(30 * scale), x + int(20 * scale), body_y0 + int(30 * scale)), fill="white", width=max(1, stroke // 2))
593    draw.line((x - int(20 * scale), body_y0 + int(48 * scale), x + int(12 * scale), body_y0 + int(48 * scale)), fill="white", width=max(1, stroke // 2))
594
595    shoulder_y = body_y0 + int(22 * scale)
596    hand_y = body_y1 - int(10 * scale)
597    if state == "excited":
598        draw.line((x - int(42 * scale), shoulder_y, x - int(88 * scale), shoulder_y - int(70 * scale)), fill="white", width=stroke)
599        draw.line((x + int(42 * scale), shoulder_y, x + int(88 * scale), shoulder_y - int(70 * scale)), fill="white", width=stroke)
600    else:
601        draw.line((x - int(42 * scale), shoulder_y, x - int(82 * scale), hand_y), fill="white", width=stroke)
602        draw.line((x + int(42 * scale), shoulder_y, x + int(82 * scale), hand_y), fill="white", width=stroke)
603    draw.arc((x - int(98 * scale), hand_y - int(18 * scale), x - int(66 * scale), hand_y + int(18 * scale)), 80, 290, fill="white", width=stroke)
604    draw.arc((x + int(66 * scale), hand_y - int(18 * scale), x + int(98 * scale), hand_y + int(18 * scale)), -110, 100, fill="white", width=stroke)
605
606    draw.line((x - int(22 * scale), body_y1, x - int(35 * scale), base_y), fill="white", width=stroke)
607    draw.line((x + int(22 * scale), body_y1, x + int(35 * scale), base_y), fill="white", width=stroke)
608    draw.rectangle((x - int(55 * scale), base_y - int(16 * scale), x - int(22 * scale), base_y), outline="white", width=stroke)
609    draw.rectangle((x + int(22 * scale), base_y - int(16 * scale), x + int(55 * scale), base_y), outline="white", width=stroke)
610
611
612def draw_sparks(draw: ImageDraw.ImageDraw, width: int, height: int, stroke: int):
613    for x, y in [(0.45, 0.42), (0.56, 0.36), (0.63, 0.48), (0.37, 0.52)]:
614        cx, cy = int(width * x), int(height * y)
615        draw.line((cx - 18, cy + 22, cx, cy - 22, cx + 18, cy + 10), fill="white", width=stroke)
616
617
618def dev_control_sprite_name(text: str) -> str:
619    if _has_any(text, ("dead", "x-eye", "x eye")):
620        return "dev_pose_dead"
621    if _has_any(text, ("collapsed", "floor", "lies flat", "flat on", "burnout")):
622        return "dev_pose_collapsed"
623    if _has_any(text, ("slumped", "exhausted", "tired")):
624        return "dev_pose_slumped"
625    if _has_any(text, ("panic", "panicked", "overwhelmed", "holy", "chaos")):
626        return "dev_pose_overwhelmed"
627    if _has_any(text, ("resurrected", "zombie")):
628        return "dev_pose_zombie"
629    if _has_any(text, ("seated", "typing", "laptop", "terminal", "nvim", "code")):
630        return "dev_seated_laptop"
631    if _has_any(text, ("excited", "wow", "arms up", "throws both arms", "celebrat")):
632        return "dev_excited_body"
633    if _has_any(text, ("coffee", "mug")):
634        return "dev_mug_body"
635    if _has_any(text, ("running", "run")):
636        return "dev_running_body"
637    return "dev_neutral_body"
638
639
640def clanker_control_sprite_name(text: str) -> str:
641    if _has_any(text, ("laptop", "typing", "interfacing")) and "clanker" in text and "dev" not in text:
642        return "clanker_at_laptop"
643    if _has_any(text, ("happy", "celebrat", "triumphant", "hyped", "success", "proud", "confident")):
644        return "clanker_happy_body"
645    return "clanker_neutral_body"
646
647
648def draw_reference_characters(
649    image: Image.Image,
650    project_root: Path,
651    text: str,
652    dev_x: float,
653    clanker_x: float,
654    base_y: int,
655    scale: float,
656) -> bool:
657    dev_name = dev_control_sprite_name(text)
658    clanker_name = clanker_control_sprite_name(text)
659    dev_sprite = control_reference_sprite(project_root, dev_name)
660    clanker_sprite = control_reference_sprite(project_root, clanker_name)
661    if dev_sprite is None or clanker_sprite is None:
662        return False
663
664    shot_scale = 1.0
665    if "close" in text:
666        shot_scale = 1.18
667    elif "wide" in text:
668        shot_scale = 0.88
669
670    dev_height = int((300 if dev_name.startswith("dev_pose_") else 330) * scale * shot_scale)
671    clanker_height = int(335 * scale * shot_scale)
672    dev_center = dev_x
673    if dev_name in {"dev_seated_laptop", "dev_laptop_body"}:
674        # These sprites include the desk/laptop, so their visual center needs to
675        # sit closer to the middle of the panel than the Dev's head alone.
676        dev_center = min(0.48, dev_x + 0.08)
677        dev_height = int(300 * scale * shot_scale)
678    if dev_name in {"dev_pose_collapsed", "dev_pose_dead"}:
679        dev_center = max(0.28, dev_x + 0.05)
680        dev_height = int(210 * scale * shot_scale)
681
682    paste_control_sprite(image, dev_sprite, int(image.width * dev_center), base_y, dev_height)
683    paste_control_sprite(image, clanker_sprite, int(image.width * clanker_x), base_y, clanker_height)
684    return True
685
686
687def make_layout_control_image(panel: PanelSpec, width: int, height: int, project_root: Path | None = None) -> Image.Image:
688    text = f"{panel.shot} {_panel_text(panel)}"
689    composition = (panel.composition or "").lower()
690    scale = min(width / 1024, height / 768)
691    stroke = max(2, int(4 * scale))
692    image = Image.new("RGBA", (width, height), (0, 0, 0, 255))
693    draw = ImageDraw.Draw(image)
694
695    dev_x = _clause_position(composition, "dev", 0.31)
696    clanker_x = _clause_position(composition, "clanker", 0.69)
697    if abs(dev_x - clanker_x) < 0.18:
698        dev_x, clanker_x = 0.31, 0.69
699
700    foreground = "foreground" in text
701    base_y = int(height * (0.86 if foreground else 0.80))
702
703    dev_sprite_name = dev_control_sprite_name(text)
704    sprite_has_laptop = dev_sprite_name in {"dev_seated_laptop", "dev_laptop_body", "dev_pose_slumped", "dev_pose_overwhelmed"}
705    if _has_any(text, ("laptop", "terminal", "monitor", "nvim", "code")) and not sprite_has_laptop:
706        draw_laptop(draw, width, height, width * 0.50, height * (0.68 if foreground else 0.70), scale, stroke)
707    if "token meter" in text or "tokens" in text:
708        draw_token_meter(draw, width, height, stroke)
709
710    used_reference_sprites = False
711    if project_root is not None:
712        used_reference_sprites = draw_reference_characters(image, project_root, text, dev_x, clanker_x, base_y, scale)
713
714    if not used_reference_sprites:
715        dev_state = "standing"
716        if _has_any(text, ("seated", "typing", "laptop")):
717            dev_state = "seated"
718        if _has_any(text, ("excited", "wow", "arms up", "throws both arms", "celebrat")):
719            dev_state = "excited"
720        if _has_any(text, ("collapsed", "floor", "dead", "lies flat", "flat on")):
721            dev_state = "collapsed"
722
723        clanker_state = "standing"
724        if _has_any(text, ("arms raised", "celebrat", "triumphant", "hyped")):
725            clanker_state = "excited"
726        if _has_any(text, ("concerned", "worried")):
727            clanker_state = "worried"
728
729        draw_dev(draw, int(width * dev_x), base_y, scale, dev_state, stroke)
730        draw_clanker(draw, int(width * clanker_x), base_y, scale, clanker_state, stroke)
731
732    if _has_any(text, ("spark", "chaos", "overclock", "erupts", "debris")):
733        draw_sparks(draw, width, height, stroke)
734
735    return image.convert("RGB")
736
737
738def fit_control_image(image: Image.Image, width: int, height: int) -> Image.Image:
739    image = ImageOps.exif_transpose(image).convert("RGB")
740    image = ImageOps.contain(image, (width, height), method=Image.Resampling.LANCZOS)
741    canvas = Image.new("RGB", (width, height), "black")
742    canvas.paste(image, ((width - image.width) // 2, (height - image.height) // 2))
743    return canvas
744
745
746def canny_control_image(image: Image.Image) -> Image.Image:
747    import cv2
748    import numpy as np
749
750    gray = np.array(ImageOps.grayscale(image))
751    edges = cv2.Canny(gray, 80, 180)
752    return Image.fromarray(edges).convert("RGB")
753
754
755def make_control_image(
756    config: GenerationConfig,
757    panel: PanelSpec,
758    project_root: Path,
759    output_path: Path,
760    panel_number: int,
761) -> Image.Image | None:
762    if not config.enableControlNet:
763        return None
764
765    if panel.controlImage:
766        control_path = (project_root / panel.controlImage).resolve()
767        if not control_path.exists():
768            raise FileNotFoundError(f"Control image not found: {control_path}")
769        image = fit_control_image(Image.open(control_path), config.width, config.height)
770        if config.controlGuideMode == "canny":
771            image = canny_control_image(image)
772    else:
773        image = make_layout_control_image(panel, config.width, config.height, project_root)
774
775    debug_path = output_path.parent / f"control-panel-{panel_number:02d}.png"
776    debug_path.parent.mkdir(parents=True, exist_ok=True)
777    image.save(debug_path)
778    return image
779
780
781def generate_panel_art(
782    comic: ComicSpec,
783    panel: PanelSpec,
784    panel_number: int,
785    project_root: Path,
786    output_path: Path,
787) -> tuple[Path, int]:
788    config = comic.generation
789    pipe = load_pipeline(config)
790
791    import torch
792
793    if hasattr(pipe, "maybe_free_model_hooks"):
794        pipe.maybe_free_model_hooks()
795    if config.device == "cuda":
796        torch.cuda.empty_cache()
797
798    reference_images = load_reference_images(config, panel, project_root)
799    execution_device = getattr(pipe, "_execution_device", config.device)
800    do_classifier_free_guidance = config.guidanceScale > 1.0
801    ip_adapter_image_embeds = make_ip_adapter_image_embeds(
802        pipe,
803        reference_images,
804        torch,
805        execution_device,
806        do_classifier_free_guidance,
807    )
808
809    seed = panel.seed if panel.seed is not None else config.seed
810    if seed is None:
811        seed = random.SystemRandom().randint(0, 2**32 - 1)
812    generator = torch.Generator(device=config.device).manual_seed(seed)
813
814    negative_prompt = config.negativePrompt
815    if panel.negativePrompt:
816        negative_prompt = f"{negative_prompt}, {panel.negativePrompt}"
817
818    call_args = {
819        "prompt": build_prompt(comic, panel, panel_number),
820        "negative_prompt": negative_prompt,
821        "width": config.width,
822        "height": config.height,
823        "num_inference_steps": config.steps,
824        "guidance_scale": config.guidanceScale,
825        "generator": generator,
826    }
827    if ip_adapter_image_embeds is not None:
828        call_args["ip_adapter_image_embeds"] = ip_adapter_image_embeds
829    else:
830        call_args["ip_adapter_image"] = make_reference_contact_sheet(reference_images)
831
832    if config.device == "cuda":
833        torch.cuda.empty_cache()
834
835    control_image = make_control_image(config, panel, project_root, output_path, panel_number)
836    if control_image is not None:
837        call_args.update(
838            {
839                "image": control_image,
840                "controlnet_conditioning_scale": panel.controlNetScale
841                if panel.controlNetScale is not None
842                else config.controlNetScale,
843                "control_guidance_start": config.controlNetStart,
844                "control_guidance_end": config.controlNetEnd,
845            }
846        )
847
848    image = pipe(**call_args).images[0]
849
850    if hasattr(pipe, "maybe_free_model_hooks"):
851        pipe.maybe_free_model_hooks()
852    if config.device == "cuda":
853        torch.cuda.empty_cache()
854
855    output_path.parent.mkdir(parents=True, exist_ok=True)
856    ImageOps.exif_transpose(image).save(output_path)
857    return output_path, seed